use of org.apache.sysml.runtime.instructions.mr.AggregateInstruction in project incubator-systemml by apache.
the class ReduceBase method processAggregateInstructions.
// process all the aggregate instructions for one group of values
protected void processAggregateInstructions(MatrixIndexes indexes, Iterator<TaggedMatrixValue> values, boolean imbededCorrection) throws IOException {
try {
while (values.hasNext()) {
TaggedMatrixValue value = values.next();
byte input = value.getTag();
ArrayList<AggregateInstruction> instructions = agg_instructions.get(input);
// if there is no specified aggregate operation on an input, by default apply sum
if (instructions == null) {
defaultAggIns.input = input;
defaultAggIns.output = input;
processAggregateHelp(indexes.getRowIndex(), indexes.getColumnIndex(), value.getBaseObject(), defaultAggIns, imbededCorrection);
} else // otherwise, perform the specified aggregate instructions
{
for (AggregateInstruction ins : instructions) processAggregateHelp(indexes.getRowIndex(), indexes.getColumnIndex(), value.getBaseObject(), ins, imbededCorrection);
}
}
} catch (Exception e) {
throw new IOException(e);
}
}
use of org.apache.sysml.runtime.instructions.mr.AggregateInstruction in project incubator-systemml by apache.
the class MatrixCharacteristics method computeDimension.
public static void computeDimension(HashMap<Byte, MatrixCharacteristics> dims, MRInstruction ins) {
MatrixCharacteristics dimOut = dims.get(ins.output);
if (dimOut == null) {
dimOut = new MatrixCharacteristics();
dims.put(ins.output, dimOut);
}
if (ins instanceof ReorgInstruction) {
ReorgInstruction realIns = (ReorgInstruction) ins;
reorg(dims.get(realIns.input), (ReorgOperator) realIns.getOperator(), dimOut);
} else if (ins instanceof AppendInstruction) {
AppendInstruction realIns = (AppendInstruction) ins;
MatrixCharacteristics in_dim1 = dims.get(realIns.input1);
MatrixCharacteristics in_dim2 = dims.get(realIns.input2);
if (realIns.isCBind())
dimOut.set(in_dim1.numRows, in_dim1.numColumns + in_dim2.numColumns, in_dim1.numRowsPerBlock, in_dim2.numColumnsPerBlock);
else
dimOut.set(in_dim1.numRows + in_dim2.numRows, in_dim1.numColumns, in_dim1.numRowsPerBlock, in_dim2.numColumnsPerBlock);
} else if (ins instanceof CumulativeAggregateInstruction) {
AggregateUnaryInstruction realIns = (AggregateUnaryInstruction) ins;
MatrixCharacteristics in = dims.get(realIns.input);
dimOut.set((long) Math.ceil((double) in.getRows() / in.getRowsPerBlock()), in.getCols(), in.getRowsPerBlock(), in.getColsPerBlock());
} else if (ins instanceof AggregateUnaryInstruction) {
AggregateUnaryInstruction realIns = (AggregateUnaryInstruction) ins;
aggregateUnary(dims.get(realIns.input), (AggregateUnaryOperator) realIns.getOperator(), dimOut);
} else if (ins instanceof AggregateBinaryInstruction) {
AggregateBinaryInstruction realIns = (AggregateBinaryInstruction) ins;
aggregateBinary(dims.get(realIns.input1), dims.get(realIns.input2), (AggregateBinaryOperator) realIns.getOperator(), dimOut);
} else if (ins instanceof MapMultChainInstruction) {
// output size independent of chain type
MapMultChainInstruction realIns = (MapMultChainInstruction) ins;
MatrixCharacteristics mc1 = dims.get(realIns.getInput1());
MatrixCharacteristics mc2 = dims.get(realIns.getInput2());
dimOut.set(mc1.numColumns, mc2.numColumns, mc1.numRowsPerBlock, mc1.numColumnsPerBlock);
} else if (ins instanceof QuaternaryInstruction) {
QuaternaryInstruction realIns = (QuaternaryInstruction) ins;
MatrixCharacteristics mc1 = dims.get(realIns.getInput1());
MatrixCharacteristics mc2 = dims.get(realIns.getInput2());
MatrixCharacteristics mc3 = dims.get(realIns.getInput3());
realIns.computeMatrixCharacteristics(mc1, mc2, mc3, dimOut);
} else if (ins instanceof ReblockInstruction) {
ReblockInstruction realIns = (ReblockInstruction) ins;
MatrixCharacteristics in_dim = dims.get(realIns.input);
dimOut.set(in_dim.numRows, in_dim.numColumns, realIns.brlen, realIns.bclen, in_dim.nonZero);
} else if (ins instanceof MatrixReshapeMRInstruction) {
MatrixReshapeMRInstruction mrinst = (MatrixReshapeMRInstruction) ins;
MatrixCharacteristics in_dim = dims.get(mrinst.input);
dimOut.set(mrinst.getNumRows(), mrinst.getNumColunms(), in_dim.getRowsPerBlock(), in_dim.getColsPerBlock(), in_dim.getNonZeros());
} else if (ins instanceof RandInstruction || ins instanceof SeqInstruction) {
DataGenMRInstruction dataIns = (DataGenMRInstruction) ins;
dimOut.set(dims.get(dataIns.getInput()));
} else if (ins instanceof ReplicateInstruction) {
ReplicateInstruction realIns = (ReplicateInstruction) ins;
realIns.computeOutputDimension(dims.get(realIns.input), dimOut);
} else if (// before unary
ins instanceof ParameterizedBuiltinMRInstruction) {
ParameterizedBuiltinMRInstruction realIns = (ParameterizedBuiltinMRInstruction) ins;
realIns.computeOutputCharacteristics(dims.get(realIns.input), dimOut);
} else if (ins instanceof ScalarInstruction || ins instanceof AggregateInstruction || (ins instanceof UnaryInstruction && !(ins instanceof MMTSJMRInstruction)) || ins instanceof ZeroOutInstruction) {
UnaryMRInstructionBase realIns = (UnaryMRInstructionBase) ins;
dimOut.set(dims.get(realIns.input));
} else if (ins instanceof MMTSJMRInstruction) {
MMTSJMRInstruction mmtsj = (MMTSJMRInstruction) ins;
MMTSJType tstype = mmtsj.getMMTSJType();
MatrixCharacteristics mc = dims.get(mmtsj.input);
dimOut.set(tstype.isLeft() ? mc.numColumns : mc.numRows, tstype.isLeft() ? mc.numColumns : mc.numRows, mc.numRowsPerBlock, mc.numColumnsPerBlock);
} else if (ins instanceof PMMJMRInstruction) {
PMMJMRInstruction pmmins = (PMMJMRInstruction) ins;
MatrixCharacteristics mc = dims.get(pmmins.input2);
dimOut.set(pmmins.getNumRows(), mc.numColumns, mc.numRowsPerBlock, mc.numColumnsPerBlock);
} else if (ins instanceof RemoveEmptyMRInstruction) {
RemoveEmptyMRInstruction realIns = (RemoveEmptyMRInstruction) ins;
MatrixCharacteristics mc = dims.get(realIns.input1);
long min = realIns.isEmptyReturn() ? 1 : 0;
if (realIns.isRemoveRows())
dimOut.set(Math.max(realIns.getOutputLen(), min), mc.getCols(), mc.numRowsPerBlock, mc.numColumnsPerBlock);
else
dimOut.set(mc.getRows(), Math.max(realIns.getOutputLen(), min), mc.numRowsPerBlock, mc.numColumnsPerBlock);
} else if (// needs to be checked before binary
ins instanceof UaggOuterChainInstruction) {
UaggOuterChainInstruction realIns = (UaggOuterChainInstruction) ins;
MatrixCharacteristics mc1 = dims.get(realIns.input1);
MatrixCharacteristics mc2 = dims.get(realIns.input2);
realIns.computeOutputCharacteristics(mc1, mc2, dimOut);
} else if (ins instanceof GroupedAggregateMInstruction) {
GroupedAggregateMInstruction realIns = (GroupedAggregateMInstruction) ins;
MatrixCharacteristics mc1 = dims.get(realIns.input1);
realIns.computeOutputCharacteristics(mc1, dimOut);
} else if (ins instanceof BinaryInstruction || ins instanceof BinaryMInstruction || ins instanceof CombineBinaryInstruction) {
BinaryMRInstructionBase realIns = (BinaryMRInstructionBase) ins;
MatrixCharacteristics mc1 = dims.get(realIns.input1);
MatrixCharacteristics mc2 = dims.get(realIns.input2);
if (mc1.getRows() > 1 && mc1.getCols() == 1 && mc2.getRows() == 1 && // outer
mc2.getCols() > 1) {
dimOut.set(mc1.getRows(), mc2.getCols(), mc1.getRowsPerBlock(), mc2.getColsPerBlock());
} else {
// default case
dimOut.set(mc1);
}
} else if (ins instanceof TernaryInstruction) {
dimOut.set(dims.get(ins.getInputIndexes()[0]));
} else if (ins instanceof CombineTernaryInstruction) {
CtableInstruction realIns = (CtableInstruction) ins;
dimOut.set(dims.get(realIns.input1));
} else if (ins instanceof CombineUnaryInstruction) {
dimOut.set(dims.get(((CombineUnaryInstruction) ins).input));
} else if (ins instanceof CM_N_COVInstruction || ins instanceof GroupedAggregateInstruction) {
dimOut.set(1, 1, 1, 1);
} else if (ins instanceof RangeBasedReIndexInstruction) {
RangeBasedReIndexInstruction realIns = (RangeBasedReIndexInstruction) ins;
MatrixCharacteristics dimIn = dims.get(realIns.input);
realIns.computeOutputCharacteristics(dimIn, dimOut);
} else if (ins instanceof CtableInstruction) {
CtableInstruction realIns = (CtableInstruction) ins;
MatrixCharacteristics in_dim = dims.get(realIns.input1);
dimOut.set(realIns.getOutputDim1(), realIns.getOutputDim2(), in_dim.numRowsPerBlock, in_dim.numColumnsPerBlock);
} else {
/*
* if ins is none of the above cases then we assume that dim_out dimensions are unknown
*/
dimOut.numRows = -1;
dimOut.numColumns = -1;
dimOut.numRowsPerBlock = 1;
dimOut.numColumnsPerBlock = 1;
}
}
use of org.apache.sysml.runtime.instructions.mr.AggregateInstruction in project systemml by apache.
the class ReduceBase method configure.
@Override
public void configure(JobConf job) {
super.configure(job);
reducerID = job.get(MRConfigurationNames.MR_TASK_ATTEMPT_ID);
dimsUnknownFilePrefix = job.get("dims.unknown.file.prefix");
// get the indexes of the final output matrices
resultIndexes = MRJobConfiguration.getResultIndexes(job);
resultDimsUnknown = MRJobConfiguration.getResultDimsUnknown(job);
// initialize SystemML Counters (defined in MRJobConfiguration)
resultsNonZeros = new long[resultIndexes.length];
resultsMaxRowDims = new long[resultIndexes.length];
resultsMaxColDims = new long[resultIndexes.length];
collectFinalMultipleOutputs = MRJobConfiguration.getMultipleConvertedOutputs(job);
// parse aggregate operations
AggregateInstruction[] agg_insts = null;
try {
agg_insts = MRJobConfiguration.getAggregateInstructions(job);
// parse unary and binary operations
MRInstruction[] tmp = MRJobConfiguration.getInstructionsInReducer(job);
if (tmp != null) {
mixed_instructions = new ArrayList<>();
Collections.addAll(mixed_instructions, tmp);
}
} catch (DMLRuntimeException e) {
throw new RuntimeException(e);
}
// load data from distributed cache (if required, reuse if jvm_reuse)
try {
setupDistCacheFiles(job);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
// reorganize the aggregate instructions, so that they are all associatied with each input
if (agg_insts != null) {
for (AggregateInstruction ins : agg_insts) {
// associate instruction to its input
ArrayList<AggregateInstruction> vec = agg_instructions.get(ins.input);
if (vec == null) {
vec = new ArrayList<>();
agg_instructions.put(ins.input, vec);
}
vec.add(ins);
if (ins.input == ins.output)
continue;
// need to add new aggregate instructions so that partial aggregation can be applied
// this is important for combiner in the reducer side
AggregateInstruction partialIns = new AggregateInstruction(ins.getOperator(), ins.output, ins.output, ins.toString());
vec = agg_instructions.get(partialIns.input);
if (vec == null) {
vec = new ArrayList<>();
agg_instructions.put(partialIns.input, vec);
}
vec.add(partialIns);
}
}
}
use of org.apache.sysml.runtime.instructions.mr.AggregateInstruction in project systemml by apache.
the class ReduceBase method processAggregateInstructions.
// process all the aggregate instructions for one group of values
protected void processAggregateInstructions(MatrixIndexes indexes, Iterator<TaggedMatrixValue> values, boolean imbededCorrection) throws IOException {
try {
while (values.hasNext()) {
TaggedMatrixValue value = values.next();
byte input = value.getTag();
ArrayList<AggregateInstruction> instructions = agg_instructions.get(input);
// if there is no specified aggregate operation on an input, by default apply sum
if (instructions == null) {
defaultAggIns.input = input;
defaultAggIns.output = input;
processAggregateHelp(indexes.getRowIndex(), indexes.getColumnIndex(), value.getBaseObject(), defaultAggIns, imbededCorrection);
} else // otherwise, perform the specified aggregate instructions
{
for (AggregateInstruction ins : instructions) processAggregateHelp(indexes.getRowIndex(), indexes.getColumnIndex(), value.getBaseObject(), ins, imbededCorrection);
}
}
} catch (Exception e) {
throw new IOException(e);
}
}
use of org.apache.sysml.runtime.instructions.mr.AggregateInstruction in project incubator-systemml by apache.
the class MMCJMRCombinerReducerBase method performAggregateInstructions.
protected MatrixValue performAggregateInstructions(TaggedFirstSecondIndexes indexes, Iterator<MatrixValue> values) throws IOException {
// manipulation on the tags first
byte realTag = indexes.getTag();
byte representTag;
if (realTag == tagForLeft)
representTag = aggBinInstruction.input1;
else
representTag = aggBinInstruction.input2;
ArrayList<AggregateInstruction> instructions = agg_instructions.get(representTag);
AggregateInstruction ins;
if (instructions == null) {
defaultAggIns.input = realTag;
defaultAggIns.output = realTag;
ins = defaultAggIns;
} else {
if (instructions.size() > 1)
throw new IOException("only one aggregate operation on input " + indexes.getTag() + " is allowed in BlockMMCJMR");
ins = instructions.get(0);
if (ins.input != ins.output)
throw new IOException("input index and output index have to be " + "the same for aggregate instructions in BlockMMCJMR");
}
// performa aggregation before doing mmcj
// TODO: customize the code, since aggregation for matrix multiplcation can only be sum
boolean needStartAgg = true;
try {
while (values.hasNext()) {
MatrixValue value = values.next();
if (needStartAgg) {
buffer.reset(value.getNumRows(), value.getNumColumns(), value.isInSparseFormat());
needStartAgg = false;
// LOG.info("initialize buffer: sparse="+buffer.isInSparseFormat()+", nonZero="+buffer.getNonZeros());
}
buffer.binaryOperationsInPlace(((AggregateOperator) ins.getOperator()).increOp, value);
// LOG.info("increment buffer: sparse="+buffer.isInSparseFormat()+", nonZero="+buffer.getNonZeros());
}
} catch (Exception e) {
throw new IOException(e);
}
if (needStartAgg)
return null;
else
return buffer;
}
Aggregations