use of org.apache.sysml.lops.MapMult in project incubator-systemml by apache.
the class AggBinaryOp method constructMRLopsMapMMWithLeftTransposeRewrite.
private Lop constructMRLopsMapMMWithLeftTransposeRewrite() {
// guaranteed to exists
Hop X = getInput().get(0).getInput().get(0);
Hop Y = getInput().get(1);
// right vector transpose CP
Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
setLineNumbers(tY);
// matrix mult
// If number of columns is smaller than block size then explicit aggregation is not required.
// i.e., entire matrix multiplication can be performed in the mappers.
boolean needAgg = (X.getDim1() <= 0 || X.getDim1() > X.getRowsInBlock());
// R disregarding transpose rewrite
boolean needPart = requiresPartitioning(MMultMethod.MAPMM_R, true);
// pre partitioning
Lop dcinput = null;
if (needPart) {
ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(Y.getDim2(), Y.getDim1(), OptimizerUtils.getSparsity(Y.getDim2(), Y.getDim1(), Y.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
ExecType.MR;
dcinput = new DataPartition(tY, DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
dcinput.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
setLineNumbers(dcinput);
} else
dcinput = tY;
MapMult mapmult = new MapMult(dcinput, X.constructLops(), getDataType(), getValueType(), false, needPart, false);
mapmult.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
setLineNumbers(mapmult);
// post aggregation
Lop mult = null;
if (needAgg) {
Group grp = new Group(mapmult, Group.OperationTypes.Sort, getDataType(), getValueType());
grp.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
setLineNumbers(grp);
Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
agg1.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
setLineNumbers(agg1);
agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
mult = agg1;
} else
mult = mapmult;
// result transpose CP
Lop out = new Transform(mult, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
out.getOutputParameters().setDimensions(X.getDim2(), Y.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
return out;
}
use of org.apache.sysml.lops.MapMult in project incubator-systemml by apache.
the class AggBinaryOp method constructSparkLopsMapMMWithLeftTransposeRewrite.
private Lop constructSparkLopsMapMMWithLeftTransposeRewrite() {
// guaranteed to exists
Hop X = getInput().get(0).getInput().get(0);
Hop Y = getInput().get(1);
// right vector transpose
Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
setLineNumbers(tY);
// matrix mult spark
boolean needAgg = requiresAggregation(MMultMethod.MAPMM_R);
SparkAggType aggtype = getSparkMMAggregationType(needAgg);
_outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
Lop mult = new MapMult(tY, X.constructLops(), getDataType(), getValueType(), false, false, _outputEmptyBlocks, aggtype);
mult.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
setLineNumbers(mult);
// result transpose (dimensions set outside)
Lop out = new Transform(mult, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
return out;
}
use of org.apache.sysml.lops.MapMult in project systemml by apache.
the class AggBinaryOp method constructMRLopsMapMM.
// ////////////////////////
// MR Lops generation
// ///////////////////////
private void constructMRLopsMapMM(MMultMethod method) {
if (method == MMultMethod.MAPMM_R && isLeftTransposeRewriteApplicable(false, true)) {
setLops(constructMRLopsMapMMWithLeftTransposeRewrite());
} else // GENERAL CASE
{
// If number of columns is smaller than block size then explicit aggregation is not required.
// i.e., entire matrix multiplication can be performed in the mappers.
boolean needAgg = requiresAggregation(method);
boolean needPart = requiresPartitioning(method, false);
_outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
// pre partitioning
Lop leftInput = getInput().get(0).constructLops();
Lop rightInput = getInput().get(1).constructLops();
if (needPart) {
if (// left in distributed cache
(method == MMultMethod.MAPMM_L)) {
Hop input = getInput().get(0);
ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(input.getDim1(), input.getDim2(), OptimizerUtils.getSparsity(input.getDim1(), input.getDim2(), input.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
ExecType.MR;
leftInput = new DataPartition(input.constructLops(), DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
leftInput.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), getRowsInBlock(), getColsInBlock(), input.getNnz());
setLineNumbers(leftInput);
} else // right side in distributed cache
{
Hop input = getInput().get(1);
ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(input.getDim1(), input.getDim2(), OptimizerUtils.getSparsity(input.getDim1(), input.getDim2(), input.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
ExecType.MR;
rightInput = new DataPartition(input.constructLops(), DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.ROW_BLOCK_WISE_N);
rightInput.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), getRowsInBlock(), getColsInBlock(), input.getNnz());
setLineNumbers(rightInput);
}
}
// core matrix mult
MapMult mapmult = new MapMult(leftInput, rightInput, getDataType(), getValueType(), (method == MMultMethod.MAPMM_R), needPart, _outputEmptyBlocks);
mapmult.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
setLineNumbers(mapmult);
// post aggregation
if (needAgg) {
Group grp = new Group(mapmult, Group.OperationTypes.Sort, getDataType(), getValueType());
Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
grp.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
agg1.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
setLineNumbers(agg1);
// aggregation uses kahanSum but the inputs do not have correction values
agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
setLops(agg1);
} else {
setLops(mapmult);
}
}
}
use of org.apache.sysml.lops.MapMult in project systemml by apache.
the class AggBinaryOp method constructSparkLopsMapMM.
private void constructSparkLopsMapMM(MMultMethod method) {
Lop mapmult = null;
if (isLeftTransposeRewriteApplicable(false, false)) {
mapmult = constructSparkLopsMapMMWithLeftTransposeRewrite();
} else {
// If number of columns is smaller than block size then explicit aggregation is not required.
// i.e., entire matrix multiplication can be performed in the mappers.
boolean needAgg = requiresAggregation(method);
SparkAggType aggtype = getSparkMMAggregationType(needAgg);
_outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
// core matrix mult
mapmult = new MapMult(getInput().get(0).constructLops(), getInput().get(1).constructLops(), getDataType(), getValueType(), (method == MMultMethod.MAPMM_R), false, _outputEmptyBlocks, aggtype);
}
setOutputDimensions(mapmult);
setLineNumbers(mapmult);
setLops(mapmult);
}
use of org.apache.sysml.lops.MapMult in project incubator-systemml by apache.
the class Dag method computeFootprintInMapper.
/**
* Computes the memory footprint required to execute <code>node</code> in the mapper.
* It is used only for those nodes that use inputs from distributed cache. The returned
* value is utilized in limiting the number of instructions piggybacked onto a single GMR mapper.
*
* @param node low-level operator
* @return memory footprint
*/
private static double computeFootprintInMapper(Lop node) {
// Memory limits must be checked only for nodes that use distributed cache
if (!node.usesDistributedCache())
// default behavior
return 0.0;
OutputParameters in1dims = node.getInputs().get(0).getOutputParameters();
OutputParameters in2dims = node.getInputs().get(1).getOutputParameters();
double footprint = 0;
if (node instanceof MapMult) {
int dcInputIndex = node.distributedCacheInputIndex()[0];
footprint = AggBinaryOp.getMapmmMemEstimate(in1dims.getNumRows(), in1dims.getNumCols(), in1dims.getRowsInBlock(), in1dims.getColsInBlock(), in1dims.getNnz(), in2dims.getNumRows(), in2dims.getNumCols(), in2dims.getRowsInBlock(), in2dims.getColsInBlock(), in2dims.getNnz(), dcInputIndex, false);
} else if (node instanceof PMMJ) {
int dcInputIndex = node.distributedCacheInputIndex()[0];
footprint = AggBinaryOp.getMapmmMemEstimate(in1dims.getNumRows(), 1, in1dims.getRowsInBlock(), in1dims.getColsInBlock(), in1dims.getNnz(), in2dims.getNumRows(), in2dims.getNumCols(), in2dims.getRowsInBlock(), in2dims.getColsInBlock(), in2dims.getNnz(), dcInputIndex, true);
} else if (node instanceof AppendM) {
footprint = BinaryOp.footprintInMapper(in1dims.getNumRows(), in1dims.getNumCols(), in2dims.getNumRows(), in2dims.getNumCols(), in1dims.getRowsInBlock(), in1dims.getColsInBlock());
} else if (node instanceof BinaryM) {
footprint = BinaryOp.footprintInMapper(in1dims.getNumRows(), in1dims.getNumCols(), in2dims.getNumRows(), in2dims.getNumCols(), in1dims.getRowsInBlock(), in1dims.getColsInBlock());
} else {
// default behavior
return 0.0;
}
return footprint;
}
Aggregations