use of org.apache.sysml.runtime.instructions.mr.AppendGInstruction in project incubator-systemml by apache.
the class MRBaseForCommonInstructions method processOneInstruction.
protected void processOneInstruction(MRInstruction ins, Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput) throws DMLRuntimeException {
if (ins instanceof AggregateBinaryInstruction) {
byte input = ((AggregateBinaryInstruction) ins).input1;
MatrixCharacteristics dim = dimensions.get(input);
if (dim == null)
throw new DMLRuntimeException("dimension for instruction " + ins + " is unset!!!");
ins.processInstruction(valueClass, cachedValues, tempValue, zeroInput, dim.getRowsPerBlock(), dim.getColsPerBlock());
} else if (ins instanceof ZeroOutInstruction || ins instanceof AggregateUnaryInstruction || ins instanceof RangeBasedReIndexInstruction || ins instanceof CumulativeSplitInstruction) {
byte input = ((UnaryMRInstructionBase) ins).input;
MatrixCharacteristics dim = dimensions.get(input);
if (dim == null)
throw new DMLRuntimeException("dimension for instruction " + ins + " is unset!!!");
if (ins instanceof CumulativeAggregateInstruction)
((CumulativeAggregateInstruction) ins).setMatrixCharacteristics(dim);
if (ins instanceof CumulativeSplitInstruction)
((CumulativeSplitInstruction) ins).setMatrixCharacteristics(dim);
ins.processInstruction(valueClass, cachedValues, tempValue, zeroInput, dim.getRowsPerBlock(), dim.getColsPerBlock());
} else if (ins instanceof ReorgInstruction) {
ReorgInstruction rinst = (ReorgInstruction) ins;
byte input = rinst.input;
MatrixCharacteristics dim = dimensions.get(input);
if (dim == null)
throw new DMLRuntimeException("dimension for instruction " + ins + " is unset!!!");
rinst.setInputMatrixCharacteristics(dim);
//MMCJMRMapper does not output empty blocks, no need to generate
rinst.setOutputEmptyBlocks(!(this instanceof MMCJMRMapper));
ins.processInstruction(valueClass, cachedValues, tempValue, zeroInput, dim.getRowsPerBlock(), dim.getColsPerBlock());
} else if (ins instanceof MatrixReshapeMRInstruction) {
MatrixReshapeMRInstruction mrins = (MatrixReshapeMRInstruction) ins;
byte input = mrins.input;
byte output = mrins.output;
MatrixCharacteristics dimIn = dimensions.get(input);
MatrixCharacteristics dimOut = dimensions.get(output);
if (dimIn == null || dimOut == null)
throw new DMLRuntimeException("dimension for instruction " + ins + " is unset!!!");
mrins.setMatrixCharacteristics(dimIn, dimOut);
mrins.processInstruction(valueClass, cachedValues, tempValue, zeroInput, dimIn.getRowsPerBlock(), dimIn.getColsPerBlock());
} else if (ins instanceof AppendMInstruction) {
byte input = ((AppendMInstruction) ins).input1;
MatrixCharacteristics dim = dimensions.get(input);
if (dim == null)
throw new DMLRuntimeException("dimension for instruction " + ins + " is unset!!!");
ins.processInstruction(valueClass, cachedValues, tempValue, zeroInput, dim.getRowsPerBlock(), dim.getColsPerBlock());
} else if (ins instanceof BinaryMInstruction || ins instanceof RemoveEmptyMRInstruction) {
byte input = ((BinaryMRInstructionBase) ins).input1;
MatrixCharacteristics dim = dimensions.get(input);
if (dim == null)
throw new DMLRuntimeException("dimension for instruction " + ins + " is unset!!!");
ins.processInstruction(valueClass, cachedValues, tempValue, zeroInput, dim.getRowsPerBlock(), dim.getColsPerBlock());
} else if (ins instanceof AppendGInstruction) {
AppendGInstruction arinst = ((AppendGInstruction) ins);
byte input = arinst.input1;
MatrixCharacteristics dimIn = dimensions.get(input);
if (dimIn == null)
throw new DMLRuntimeException("Dimensions for instruction " + arinst + " is unset!!!");
arinst.processInstruction(valueClass, cachedValues, tempValue, zeroInput, dimIn.getRowsPerBlock(), dimIn.getColsPerBlock());
} else if (ins instanceof UnaryMRInstructionBase) {
UnaryMRInstructionBase rinst = (UnaryMRInstructionBase) ins;
MatrixCharacteristics dimIn = dimensions.get(rinst.input);
if (dimIn == null)
throw new DMLRuntimeException("Dimensions for instruction " + rinst + " is unset!!!");
rinst.processInstruction(valueClass, cachedValues, tempValue, zeroInput, dimIn.getRowsPerBlock(), dimIn.getColsPerBlock());
} else if (ins instanceof BinaryMRInstructionBase) {
BinaryMRInstructionBase rinst = (BinaryMRInstructionBase) ins;
MatrixCharacteristics dimIn = dimensions.get(rinst.input1);
if (//not set for all
dimIn != null)
rinst.processInstruction(valueClass, cachedValues, tempValue, zeroInput, dimIn.getRowsPerBlock(), dimIn.getColsPerBlock());
else
ins.processInstruction(valueClass, cachedValues, tempValue, zeroInput, -1, -1);
} else
ins.processInstruction(valueClass, cachedValues, tempValue, zeroInput, -1, -1);
//System.out.println(ins.getMRInstructionType()+" in "+time.stop());
}
use of org.apache.sysml.runtime.instructions.mr.AppendGInstruction in project incubator-systemml by apache.
the class MRJobConfiguration method computeMatrixCharacteristics.
/**
* NOTE: this method needs to be in-sync with MRBaseForCommonInstructions.processOneInstruction,
* otherwise, the latter will potentially fail with missing dimension information.
*
* @param job job configuration
* @param inputIndexes array of byte indexes
* @param dataGenInstructions data gen instructions as a string
* @param instructionsInMapper instruction in mapper as a string
* @param reblockInstructions reblock instructions as a string
* @param aggInstructionsInReducer aggregate instructions in reducer as a string
* @param aggBinInstructions binary aggregate instructions as a string
* @param otherInstructionsInReducer other instructions in reducer as a string
* @param resultIndexes array of byte result indexes
* @param mapOutputIndexes set of map output indexes
* @param forMMCJ ?
* @return reducer groups
* @throws DMLRuntimeException if DMLRuntimeException occurs
*/
public static MatrixChar_N_ReducerGroups computeMatrixCharacteristics(JobConf job, byte[] inputIndexes, String dataGenInstructions, String instructionsInMapper, String reblockInstructions, String aggInstructionsInReducer, String aggBinInstructions, String otherInstructionsInReducer, byte[] resultIndexes, HashSet<Byte> mapOutputIndexes, boolean forMMCJ) throws DMLRuntimeException {
HashSet<Byte> intermediateMatrixIndexes = new HashSet<Byte>();
HashMap<Byte, MatrixCharacteristics> dims = new HashMap<Byte, MatrixCharacteristics>();
for (byte i : inputIndexes) {
MatrixCharacteristics dim = new MatrixCharacteristics(getNumRows(job, i), getNumColumns(job, i), getNumRowsPerBlock(job, i), getNumColumnsPerBlock(job, i), getNumNonZero(job, i));
dims.put(i, dim);
}
DataGenMRInstruction[] dataGenIns = null;
dataGenIns = MRInstructionParser.parseDataGenInstructions(dataGenInstructions);
if (dataGenIns != null) {
for (DataGenMRInstruction ins : dataGenIns) {
MatrixCharacteristics.computeDimension(dims, ins);
}
}
MRInstruction[] insMapper = MRInstructionParser.parseMixedInstructions(instructionsInMapper);
if (insMapper != null) {
for (MRInstruction ins : insMapper) {
MatrixCharacteristics.computeDimension(dims, ins);
if (ins instanceof UnaryMRInstructionBase) {
UnaryMRInstructionBase tempIns = (UnaryMRInstructionBase) ins;
setIntermediateMatrixCharactristics(job, tempIns.input, dims.get(tempIns.input));
intermediateMatrixIndexes.add(tempIns.input);
} else if (ins instanceof AppendMInstruction) {
AppendMInstruction tempIns = (AppendMInstruction) ins;
setIntermediateMatrixCharactristics(job, tempIns.input1, dims.get(tempIns.input1));
intermediateMatrixIndexes.add(tempIns.input1);
} else if (ins instanceof AppendGInstruction) {
AppendGInstruction tempIns = (AppendGInstruction) ins;
setIntermediateMatrixCharactristics(job, tempIns.input1, dims.get(tempIns.input1));
intermediateMatrixIndexes.add(tempIns.input1);
} else if (ins instanceof BinaryMInstruction) {
BinaryMInstruction tempIns = (BinaryMInstruction) ins;
setIntermediateMatrixCharactristics(job, tempIns.input1, dims.get(tempIns.input1));
intermediateMatrixIndexes.add(tempIns.input1);
} else if (ins instanceof AggregateBinaryInstruction) {
AggregateBinaryInstruction tempIns = (AggregateBinaryInstruction) ins;
setIntermediateMatrixCharactristics(job, tempIns.input1, dims.get(tempIns.input1));
//TODO
intermediateMatrixIndexes.add(tempIns.input1);
} else if (ins instanceof MapMultChainInstruction) {
MapMultChainInstruction tempIns = (MapMultChainInstruction) ins;
setIntermediateMatrixCharactristics(job, tempIns.getInput1(), dims.get(tempIns.getInput2()));
intermediateMatrixIndexes.add(tempIns.getInput1());
} else if (ins instanceof PMMJMRInstruction) {
PMMJMRInstruction tempIns = (PMMJMRInstruction) ins;
setIntermediateMatrixCharactristics(job, tempIns.input2, dims.get(tempIns.input2));
intermediateMatrixIndexes.add(tempIns.input2);
}
}
}
ReblockInstruction[] reblockIns = MRInstructionParser.parseReblockInstructions(reblockInstructions);
if (reblockIns != null) {
for (ReblockInstruction ins : reblockIns) {
MatrixCharacteristics.computeDimension(dims, ins);
setMatrixCharactristicsForReblock(job, ins.output, dims.get(ins.output));
}
}
Instruction[] aggIns = MRInstructionParser.parseAggregateInstructions(aggInstructionsInReducer);
if (aggIns != null) {
for (Instruction ins : aggIns) {
MatrixCharacteristics.computeDimension(dims, (MRInstruction) ins);
// if instruction's output is not in resultIndexes, then add its dimensions to jobconf
MRInstruction mrins = (MRInstruction) ins;
boolean found = false;
for (byte b : resultIndexes) {
if (b == mrins.output) {
found = true;
break;
}
}
if (!found) {
setIntermediateMatrixCharactristics(job, mrins.output, dims.get(mrins.output));
intermediateMatrixIndexes.add(mrins.output);
}
}
}
long numReduceGroups = 0;
AggregateBinaryInstruction[] aggBinIns = getAggregateBinaryInstructions(job);
if (aggBinIns != null) {
for (AggregateBinaryInstruction ins : aggBinIns) {
MatrixCharacteristics dim1 = dims.get(ins.input1);
MatrixCharacteristics dim2 = dims.get(ins.input2);
setMatrixCharactristicsForBinAgg(job, ins.input1, dim1);
setMatrixCharactristicsForBinAgg(job, ins.input2, dim2);
MatrixCharacteristics.computeDimension(dims, ins);
if (//there will be only one aggbin operation for MMCJ
forMMCJ)
numReduceGroups = (long) Math.ceil((double) dim1.getCols() / (double) dim1.getColsPerBlock());
}
}
if (!forMMCJ) {
//store the skylines
ArrayList<Long> xs = new ArrayList<Long>(mapOutputIndexes.size());
ArrayList<Long> ys = new ArrayList<Long>(mapOutputIndexes.size());
for (byte idx : mapOutputIndexes) {
MatrixCharacteristics dim = dims.get(idx);
long x = (long) Math.ceil((double) dim.getRows() / (double) dim.getRowsPerBlock());
long y = (long) Math.ceil((double) dim.getCols() / (double) dim.getColsPerBlock());
int i = 0;
boolean toadd = true;
while (i < xs.size()) {
if ((x >= xs.get(i) && y > ys.get(i)) || (x > xs.get(i) && y >= ys.get(i))) {
//remove any included x's and y's
xs.remove(i);
ys.remove(i);
} else if (//if included in others, stop
x <= xs.get(i) && y <= ys.get(i)) {
toadd = false;
break;
} else
i++;
}
if (toadd) {
xs.add(x);
ys.add(y);
}
}
//sort by x
TreeMap<Long, Long> map = new TreeMap<Long, Long>();
for (int i = 0; i < xs.size(); i++) map.put(xs.get(i), ys.get(i));
numReduceGroups = 0;
//compute area
long prev = 0;
for (Entry<Long, Long> e : map.entrySet()) {
numReduceGroups += (e.getKey() - prev) * e.getValue();
prev = e.getKey();
}
}
MRInstruction[] insReducer = MRInstructionParser.parseMixedInstructions(otherInstructionsInReducer);
if (insReducer != null) {
for (MRInstruction ins : insReducer) {
MatrixCharacteristics.computeDimension(dims, ins);
if (ins instanceof UnaryMRInstructionBase) {
UnaryMRInstructionBase tempIns = (UnaryMRInstructionBase) ins;
setIntermediateMatrixCharactristics(job, tempIns.input, dims.get(tempIns.input));
intermediateMatrixIndexes.add(tempIns.input);
} else if (ins instanceof RemoveEmptyMRInstruction) {
RemoveEmptyMRInstruction tempIns = (RemoveEmptyMRInstruction) ins;
setIntermediateMatrixCharactristics(job, tempIns.input1, dims.get(tempIns.input1));
intermediateMatrixIndexes.add(tempIns.input1);
}
// if instruction's output is not in resultIndexes, then add its dimensions to jobconf
boolean found = false;
for (byte b : resultIndexes) {
if (b == ins.output) {
found = true;
break;
}
}
if (!found) {
setIntermediateMatrixCharactristics(job, ins.output, dims.get(ins.output));
intermediateMatrixIndexes.add(ins.output);
}
}
}
setIntermediateMatrixIndexes(job, intermediateMatrixIndexes);
for (byte tag : mapOutputIndexes) setMatrixCharactristicsForMapperOutput(job, tag, dims.get(tag));
MatrixCharacteristics[] stats = new MatrixCharacteristics[resultIndexes.length];
MatrixCharacteristics resultDims;
for (int i = 0; i < resultIndexes.length; i++) {
resultDims = dims.get(resultIndexes[i]);
stats[i] = resultDims;
setMatrixCharactristicsForOutput(job, resultIndexes[i], stats[i]);
}
return new MatrixChar_N_ReducerGroups(stats, numReduceGroups);
}
Aggregations