use of org.apache.sysml.runtime.instructions.mr.RandInstruction in project incubator-systemml by apache.
the class Recompiler method checkCPDataGen.
public static boolean checkCPDataGen(MRJobInstruction inst, String updatedRandInst) {
boolean ret = true;
// check only shuffle inst
String shuffleInst = inst.getIv_shuffleInstructions();
String rrInst = inst.getIv_recordReaderInstructions();
String mapInst = inst.getIv_instructionsInMapper();
String aggInst = inst.getIv_aggInstructions();
String otherInst = inst.getIv_otherInstructions();
if ((shuffleInst != null && shuffleInst.length() > 0) || (rrInst != null && rrInst.length() > 0) || (mapInst != null && mapInst.length() > 0) || (aggInst != null && aggInst.length() > 0) || (otherInst != null && otherInst.length() > 0)) {
ret = false;
}
// check only rand inst
if (ret) {
String[] instParts = updatedRandInst.split(Lop.INSTRUCTION_DELIMITOR);
for (String lrandStr : instParts) {
if (InstructionUtils.getOpCode(lrandStr).equals(DataGen.RAND_OPCODE)) {
// check recompile memory budget
RandInstruction lrandInst = (RandInstruction) RandInstruction.parseInstruction(lrandStr);
long rows = lrandInst.getRows();
long cols = lrandInst.getCols();
double sparsity = lrandInst.getSparsity();
double mem = MatrixBlock.estimateSizeInMemory(rows, cols, sparsity);
if (!OptimizerUtils.isValidCPDimensions(rows, cols) || !OptimizerUtils.isValidCPMatrixSize(rows, cols, sparsity) || mem >= OptimizerUtils.getLocalMemBudget()) {
ret = false;
break;
}
} else if (InstructionUtils.getOpCode(lrandStr).equals(DataGen.SEQ_OPCODE)) {
// check recompile memory budget
// (don't account for sparsity because always dense)
SeqInstruction lrandInst = (SeqInstruction) SeqInstruction.parseInstruction(lrandStr);
long rows = lrandInst.getRows();
long cols = lrandInst.getCols();
double mem = MatrixBlock.estimateSizeInMemory(rows, cols, 1.0d);
if (!OptimizerUtils.isValidCPDimensions(rows, cols) || !OptimizerUtils.isValidCPMatrixSize(rows, cols, 1.0d) || mem >= OptimizerUtils.getLocalMemBudget()) {
ret = false;
break;
}
} else {
ret = false;
break;
}
}
}
return ret;
}
use of org.apache.sysml.runtime.instructions.mr.RandInstruction in project incubator-systemml by apache.
the class MatrixCharacteristics method computeDimension.
public static void computeDimension(HashMap<Byte, MatrixCharacteristics> dims, MRInstruction ins) {
MatrixCharacteristics dimOut = dims.get(ins.output);
if (dimOut == null) {
dimOut = new MatrixCharacteristics();
dims.put(ins.output, dimOut);
}
if (ins instanceof ReorgInstruction) {
ReorgInstruction realIns = (ReorgInstruction) ins;
reorg(dims.get(realIns.input), (ReorgOperator) realIns.getOperator(), dimOut);
} else if (ins instanceof AppendInstruction) {
AppendInstruction realIns = (AppendInstruction) ins;
MatrixCharacteristics in_dim1 = dims.get(realIns.input1);
MatrixCharacteristics in_dim2 = dims.get(realIns.input2);
if (realIns.isCBind())
dimOut.set(in_dim1.numRows, in_dim1.numColumns + in_dim2.numColumns, in_dim1.numRowsPerBlock, in_dim2.numColumnsPerBlock);
else
dimOut.set(in_dim1.numRows + in_dim2.numRows, in_dim1.numColumns, in_dim1.numRowsPerBlock, in_dim2.numColumnsPerBlock);
} else if (ins instanceof CumulativeAggregateInstruction) {
AggregateUnaryInstruction realIns = (AggregateUnaryInstruction) ins;
MatrixCharacteristics in = dims.get(realIns.input);
dimOut.set((long) Math.ceil((double) in.getRows() / in.getRowsPerBlock()), in.getCols(), in.getRowsPerBlock(), in.getColsPerBlock());
} else if (ins instanceof AggregateUnaryInstruction) {
AggregateUnaryInstruction realIns = (AggregateUnaryInstruction) ins;
aggregateUnary(dims.get(realIns.input), (AggregateUnaryOperator) realIns.getOperator(), dimOut);
} else if (ins instanceof AggregateBinaryInstruction) {
AggregateBinaryInstruction realIns = (AggregateBinaryInstruction) ins;
aggregateBinary(dims.get(realIns.input1), dims.get(realIns.input2), (AggregateBinaryOperator) realIns.getOperator(), dimOut);
} else if (ins instanceof MapMultChainInstruction) {
// output size independent of chain type
MapMultChainInstruction realIns = (MapMultChainInstruction) ins;
MatrixCharacteristics mc1 = dims.get(realIns.getInput1());
MatrixCharacteristics mc2 = dims.get(realIns.getInput2());
dimOut.set(mc1.numColumns, mc2.numColumns, mc1.numRowsPerBlock, mc1.numColumnsPerBlock);
} else if (ins instanceof QuaternaryInstruction) {
QuaternaryInstruction realIns = (QuaternaryInstruction) ins;
MatrixCharacteristics mc1 = dims.get(realIns.getInput1());
MatrixCharacteristics mc2 = dims.get(realIns.getInput2());
MatrixCharacteristics mc3 = dims.get(realIns.getInput3());
realIns.computeMatrixCharacteristics(mc1, mc2, mc3, dimOut);
} else if (ins instanceof ReblockInstruction) {
ReblockInstruction realIns = (ReblockInstruction) ins;
MatrixCharacteristics in_dim = dims.get(realIns.input);
dimOut.set(in_dim.numRows, in_dim.numColumns, realIns.brlen, realIns.bclen, in_dim.nonZero);
} else if (ins instanceof MatrixReshapeMRInstruction) {
MatrixReshapeMRInstruction mrinst = (MatrixReshapeMRInstruction) ins;
MatrixCharacteristics in_dim = dims.get(mrinst.input);
dimOut.set(mrinst.getNumRows(), mrinst.getNumColunms(), in_dim.getRowsPerBlock(), in_dim.getColsPerBlock(), in_dim.getNonZeros());
} else if (ins instanceof RandInstruction || ins instanceof SeqInstruction) {
DataGenMRInstruction dataIns = (DataGenMRInstruction) ins;
dimOut.set(dims.get(dataIns.getInput()));
} else if (ins instanceof ReplicateInstruction) {
ReplicateInstruction realIns = (ReplicateInstruction) ins;
realIns.computeOutputDimension(dims.get(realIns.input), dimOut);
} else if (// before unary
ins instanceof ParameterizedBuiltinMRInstruction) {
ParameterizedBuiltinMRInstruction realIns = (ParameterizedBuiltinMRInstruction) ins;
realIns.computeOutputCharacteristics(dims.get(realIns.input), dimOut);
} else if (ins instanceof ScalarInstruction || ins instanceof AggregateInstruction || (ins instanceof UnaryInstruction && !(ins instanceof MMTSJMRInstruction)) || ins instanceof ZeroOutInstruction) {
UnaryMRInstructionBase realIns = (UnaryMRInstructionBase) ins;
dimOut.set(dims.get(realIns.input));
} else if (ins instanceof MMTSJMRInstruction) {
MMTSJMRInstruction mmtsj = (MMTSJMRInstruction) ins;
MMTSJType tstype = mmtsj.getMMTSJType();
MatrixCharacteristics mc = dims.get(mmtsj.input);
dimOut.set(tstype.isLeft() ? mc.numColumns : mc.numRows, tstype.isLeft() ? mc.numColumns : mc.numRows, mc.numRowsPerBlock, mc.numColumnsPerBlock);
} else if (ins instanceof PMMJMRInstruction) {
PMMJMRInstruction pmmins = (PMMJMRInstruction) ins;
MatrixCharacteristics mc = dims.get(pmmins.input2);
dimOut.set(pmmins.getNumRows(), mc.numColumns, mc.numRowsPerBlock, mc.numColumnsPerBlock);
} else if (ins instanceof RemoveEmptyMRInstruction) {
RemoveEmptyMRInstruction realIns = (RemoveEmptyMRInstruction) ins;
MatrixCharacteristics mc = dims.get(realIns.input1);
long min = realIns.isEmptyReturn() ? 1 : 0;
if (realIns.isRemoveRows())
dimOut.set(Math.max(realIns.getOutputLen(), min), mc.getCols(), mc.numRowsPerBlock, mc.numColumnsPerBlock);
else
dimOut.set(mc.getRows(), Math.max(realIns.getOutputLen(), min), mc.numRowsPerBlock, mc.numColumnsPerBlock);
} else if (// needs to be checked before binary
ins instanceof UaggOuterChainInstruction) {
UaggOuterChainInstruction realIns = (UaggOuterChainInstruction) ins;
MatrixCharacteristics mc1 = dims.get(realIns.input1);
MatrixCharacteristics mc2 = dims.get(realIns.input2);
realIns.computeOutputCharacteristics(mc1, mc2, dimOut);
} else if (ins instanceof GroupedAggregateMInstruction) {
GroupedAggregateMInstruction realIns = (GroupedAggregateMInstruction) ins;
MatrixCharacteristics mc1 = dims.get(realIns.input1);
realIns.computeOutputCharacteristics(mc1, dimOut);
} else if (ins instanceof BinaryInstruction || ins instanceof BinaryMInstruction || ins instanceof CombineBinaryInstruction) {
BinaryMRInstructionBase realIns = (BinaryMRInstructionBase) ins;
MatrixCharacteristics mc1 = dims.get(realIns.input1);
MatrixCharacteristics mc2 = dims.get(realIns.input2);
if (mc1.getRows() > 1 && mc1.getCols() == 1 && mc2.getRows() == 1 && // outer
mc2.getCols() > 1) {
dimOut.set(mc1.getRows(), mc2.getCols(), mc1.getRowsPerBlock(), mc2.getColsPerBlock());
} else {
// default case
dimOut.set(mc1);
}
} else if (ins instanceof TernaryInstruction) {
dimOut.set(dims.get(ins.getInputIndexes()[0]));
} else if (ins instanceof CombineTernaryInstruction) {
CtableInstruction realIns = (CtableInstruction) ins;
dimOut.set(dims.get(realIns.input1));
} else if (ins instanceof CombineUnaryInstruction) {
dimOut.set(dims.get(((CombineUnaryInstruction) ins).input));
} else if (ins instanceof CM_N_COVInstruction || ins instanceof GroupedAggregateInstruction) {
dimOut.set(1, 1, 1, 1);
} else if (ins instanceof RangeBasedReIndexInstruction) {
RangeBasedReIndexInstruction realIns = (RangeBasedReIndexInstruction) ins;
MatrixCharacteristics dimIn = dims.get(realIns.input);
realIns.computeOutputCharacteristics(dimIn, dimOut);
} else if (ins instanceof CtableInstruction) {
CtableInstruction realIns = (CtableInstruction) ins;
MatrixCharacteristics in_dim = dims.get(realIns.input1);
dimOut.set(realIns.getOutputDim1(), realIns.getOutputDim2(), in_dim.numRowsPerBlock, in_dim.numColumnsPerBlock);
} else {
/*
* if ins is none of the above cases then we assume that dim_out dimensions are unknown
*/
dimOut.numRows = -1;
dimOut.numColumns = -1;
dimOut.numRowsPerBlock = 1;
dimOut.numColumnsPerBlock = 1;
}
}
use of org.apache.sysml.runtime.instructions.mr.RandInstruction in project systemml by apache.
the class Recompiler method checkCPDataGen.
public static boolean checkCPDataGen(MRJobInstruction inst, String updatedRandInst) {
boolean ret = true;
// check only shuffle inst
String shuffleInst = inst.getIv_shuffleInstructions();
String rrInst = inst.getIv_recordReaderInstructions();
String mapInst = inst.getIv_instructionsInMapper();
String aggInst = inst.getIv_aggInstructions();
String otherInst = inst.getIv_otherInstructions();
if ((shuffleInst != null && shuffleInst.length() > 0) || (rrInst != null && rrInst.length() > 0) || (mapInst != null && mapInst.length() > 0) || (aggInst != null && aggInst.length() > 0) || (otherInst != null && otherInst.length() > 0)) {
ret = false;
}
// check only rand inst
if (ret) {
String[] instParts = updatedRandInst.split(Lop.INSTRUCTION_DELIMITOR);
for (String lrandStr : instParts) {
if (InstructionUtils.getOpCode(lrandStr).equals(DataGen.RAND_OPCODE)) {
// check recompile memory budget
RandInstruction lrandInst = (RandInstruction) RandInstruction.parseInstruction(lrandStr);
long rows = lrandInst.getRows();
long cols = lrandInst.getCols();
double sparsity = lrandInst.getSparsity();
double mem = MatrixBlock.estimateSizeInMemory(rows, cols, sparsity);
if (!OptimizerUtils.isValidCPDimensions(rows, cols) || !OptimizerUtils.isValidCPMatrixSize(rows, cols, sparsity) || mem >= OptimizerUtils.getLocalMemBudget()) {
ret = false;
break;
}
} else if (InstructionUtils.getOpCode(lrandStr).equals(DataGen.SEQ_OPCODE)) {
// check recompile memory budget
// (don't account for sparsity because always dense)
SeqInstruction lrandInst = (SeqInstruction) SeqInstruction.parseInstruction(lrandStr);
long rows = lrandInst.getRows();
long cols = lrandInst.getCols();
double mem = MatrixBlock.estimateSizeInMemory(rows, cols, 1.0d);
if (!OptimizerUtils.isValidCPDimensions(rows, cols) || !OptimizerUtils.isValidCPMatrixSize(rows, cols, 1.0d) || mem >= OptimizerUtils.getLocalMemBudget()) {
ret = false;
break;
}
} else {
ret = false;
break;
}
}
}
return ret;
}
use of org.apache.sysml.runtime.instructions.mr.RandInstruction in project incubator-systemml by apache.
the class RunMRJobs method executeInMemoryDataGenOperations.
private static JobReturn executeInMemoryDataGenOperations(MRJobInstruction inst, String randInst, MatrixObject[] outputMatrices) {
MatrixCharacteristics[] mc = new MatrixCharacteristics[outputMatrices.length];
DataGenMRInstruction[] dgSet = MRInstructionParser.parseDataGenInstructions(randInst);
byte[] results = inst.getIv_resultIndices();
for (DataGenMRInstruction ldgInst : dgSet) {
if (ldgInst instanceof RandInstruction) {
// CP Rand block operation
RandInstruction lrand = (RandInstruction) ldgInst;
RandomMatrixGenerator rgen = LibMatrixDatagen.createRandomMatrixGenerator(lrand.getProbabilityDensityFunction(), (int) lrand.getRows(), (int) lrand.getCols(), lrand.getRowsInBlock(), lrand.getColsInBlock(), lrand.getSparsity(), lrand.getMinValue(), lrand.getMaxValue(), lrand.getPdfParams());
MatrixBlock mb = MatrixBlock.randOperations(rgen, lrand.getSeed());
for (int i = 0; i < results.length; i++) if (lrand.output == results[i]) {
outputMatrices[i].acquireModify(mb);
outputMatrices[i].release();
mc[i] = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), lrand.getRowsInBlock(), lrand.getColsInBlock(), mb.getNonZeros());
}
} else if (ldgInst instanceof SeqInstruction) {
SeqInstruction lseq = (SeqInstruction) ldgInst;
MatrixBlock mb = MatrixBlock.seqOperations(lseq.fromValue, lseq.toValue, lseq.incrValue);
for (int i = 0; i < results.length; i++) if (lseq.output == results[i]) {
outputMatrices[i].acquireModify(mb);
outputMatrices[i].release();
mc[i] = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), lseq.getRowsInBlock(), lseq.getColsInBlock(), mb.getNonZeros());
}
}
}
return new JobReturn(mc, inst.getOutputInfos(), true);
}
use of org.apache.sysml.runtime.instructions.mr.RandInstruction in project incubator-systemml by apache.
the class DataGenMR method runJob.
/**
* <p>Starts a Rand MapReduce job which will produce one or more random objects.</p>
*
* @param inst MR job instruction
* @param dataGenInstructions array of data gen instructions
* @param instructionsInMapper instructions in mapper
* @param aggInstructionsInReducer aggregate instructions in reducer
* @param otherInstructionsInReducer other instructions in reducer
* @param numReducers number of reducers
* @param replication file replication
* @param resultIndexes result indexes for each random object
* @param dimsUnknownFilePrefix file path prefix when dimensions unknown
* @param outputs output file for each random object
* @param outputInfos output information for each random object
* @return matrix characteristics for each random object
* @throws Exception if Exception occurs
*/
public static JobReturn runJob(MRJobInstruction inst, String[] dataGenInstructions, String instructionsInMapper, String aggInstructionsInReducer, String otherInstructionsInReducer, int numReducers, int replication, byte[] resultIndexes, String dimsUnknownFilePrefix, String[] outputs, OutputInfo[] outputInfos) throws Exception {
JobConf job = new JobConf(DataGenMR.class);
job.setJobName("DataGen-MR");
// whether use block representation or cell representation
MRJobConfiguration.setMatrixValueClass(job, true);
byte[] realIndexes = new byte[dataGenInstructions.length];
for (byte b = 0; b < realIndexes.length; b++) realIndexes[b] = b;
String[] inputs = new String[dataGenInstructions.length];
InputInfo[] inputInfos = new InputInfo[dataGenInstructions.length];
long[] rlens = new long[dataGenInstructions.length];
long[] clens = new long[dataGenInstructions.length];
int[] brlens = new int[dataGenInstructions.length];
int[] bclens = new int[dataGenInstructions.length];
FileSystem fs = FileSystem.get(job);
String dataGenInsStr = "";
int numblocks = 0;
int maxbrlen = -1, maxbclen = -1;
double maxsparsity = -1;
for (int i = 0; i < dataGenInstructions.length; i++) {
dataGenInsStr = dataGenInsStr + Lop.INSTRUCTION_DELIMITOR + dataGenInstructions[i];
MRInstruction mrins = MRInstructionParser.parseSingleInstruction(dataGenInstructions[i]);
MRType mrtype = mrins.getMRInstructionType();
DataGenMRInstruction genInst = (DataGenMRInstruction) mrins;
rlens[i] = genInst.getRows();
clens[i] = genInst.getCols();
brlens[i] = genInst.getRowsInBlock();
bclens[i] = genInst.getColsInBlock();
maxbrlen = Math.max(maxbrlen, brlens[i]);
maxbclen = Math.max(maxbclen, bclens[i]);
if (mrtype == MRType.Rand) {
RandInstruction randInst = (RandInstruction) mrins;
inputs[i] = LibMatrixDatagen.generateUniqueSeedPath(genInst.getBaseDir());
maxsparsity = Math.max(maxsparsity, randInst.getSparsity());
PrintWriter pw = null;
try {
pw = new PrintWriter(fs.create(new Path(inputs[i])));
// for obj reuse and preventing repeated buffer re-allocations
StringBuilder sb = new StringBuilder();
// seed generation
Well1024a bigrand = LibMatrixDatagen.setupSeedsForRand(randInst.getSeed());
for (long r = 0; r < Math.max(rlens[i], 1); r += brlens[i]) {
long curBlockRowSize = Math.min(brlens[i], (rlens[i] - r));
for (long c = 0; c < Math.max(clens[i], 1); c += bclens[i]) {
long curBlockColSize = Math.min(bclens[i], (clens[i] - c));
sb.append((r / brlens[i]) + 1);
sb.append(',');
sb.append((c / bclens[i]) + 1);
sb.append(',');
sb.append(curBlockRowSize);
sb.append(',');
sb.append(curBlockColSize);
sb.append(',');
sb.append(bigrand.nextLong());
pw.println(sb.toString());
sb.setLength(0);
numblocks++;
}
}
} finally {
IOUtilFunctions.closeSilently(pw);
}
inputInfos[i] = InputInfo.TextCellInputInfo;
} else if (mrtype == MRType.Seq) {
SeqInstruction seqInst = (SeqInstruction) mrins;
inputs[i] = genInst.getBaseDir() + System.currentTimeMillis() + ".seqinput";
// always dense
maxsparsity = 1.0;
double from = seqInst.fromValue;
double to = seqInst.toValue;
double incr = seqInst.incrValue;
// handle default 1 to -1 for special case of from>to
incr = LibMatrixDatagen.updateSeqIncr(from, to, incr);
// Correctness checks on (from, to, incr)
boolean neg = (from > to);
if (incr == 0)
throw new DMLRuntimeException("Invalid value for \"increment\" in seq().");
if (neg != (incr < 0))
throw new DMLRuntimeException("Wrong sign for the increment in a call to seq()");
// Compute the number of rows in the sequence
long numrows = UtilFunctions.getSeqLength(from, to, incr);
if (rlens[i] > 0) {
if (numrows != rlens[i])
throw new DMLRuntimeException("Unexpected error while processing sequence instruction. Expected number of rows does not match given number: " + rlens[i] + " != " + numrows);
} else {
rlens[i] = numrows;
}
if (clens[i] > 0 && clens[i] != 1)
throw new DMLRuntimeException("Unexpected error while processing sequence instruction. Number of columns (" + clens[i] + ") must be equal to 1.");
else
clens[i] = 1;
PrintWriter pw = null;
try {
pw = new PrintWriter(fs.create(new Path(inputs[i])));
StringBuilder sb = new StringBuilder();
double temp = from;
double block_from, block_to;
for (long r = 0; r < rlens[i]; r += brlens[i]) {
long curBlockRowSize = Math.min(brlens[i], (rlens[i] - r));
// block (bid_i,bid_j) generates a sequence from the interval [block_from, block_to] (inclusive of both end points of the interval)
long bid_i = ((r / brlens[i]) + 1);
long bid_j = 1;
block_from = temp;
block_to = temp + (curBlockRowSize - 1) * incr;
// next block starts from here
temp = block_to + incr;
sb.append(bid_i);
sb.append(',');
sb.append(bid_j);
sb.append(',');
sb.append(block_from);
sb.append(',');
sb.append(block_to);
sb.append(',');
sb.append(incr);
pw.println(sb.toString());
sb.setLength(0);
numblocks++;
}
} finally {
IOUtilFunctions.closeSilently(pw);
}
inputInfos[i] = InputInfo.TextCellInputInfo;
} else {
throw new DMLRuntimeException("Unexpected Data Generation Instruction Type: " + mrtype);
}
}
// remove the first ","
dataGenInsStr = dataGenInsStr.substring(1);
RunningJob runjob;
MatrixCharacteristics[] stats;
try {
// set up the block size
MRJobConfiguration.setBlocksSizes(job, realIndexes, brlens, bclens);
// set up the input files and their format information
MRJobConfiguration.setUpMultipleInputs(job, realIndexes, inputs, inputInfos, brlens, bclens, false, ConvertTarget.BLOCK);
// set up the dimensions of input matrices
MRJobConfiguration.setMatricesDimensions(job, realIndexes, rlens, clens);
MRJobConfiguration.setDimsUnknownFilePrefix(job, dimsUnknownFilePrefix);
// set up the block size
MRJobConfiguration.setBlocksSizes(job, realIndexes, brlens, bclens);
// set up the rand Instructions
MRJobConfiguration.setRandInstructions(job, dataGenInsStr);
// set up unary instructions that will perform in the mapper
MRJobConfiguration.setInstructionsInMapper(job, instructionsInMapper);
// set up the aggregate instructions that will happen in the combiner and reducer
MRJobConfiguration.setAggregateInstructions(job, aggInstructionsInReducer);
// set up the instructions that will happen in the reducer, after the aggregation instrucions
MRJobConfiguration.setInstructionsInReducer(job, otherInstructionsInReducer);
// set up the replication factor for the results
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
// set up map/reduce memory configurations (if in AM context)
DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
// set up custom map/reduce configurations
MRJobConfiguration.setupCustomMRConfigurations(job, config);
// determine degree of parallelism (nmappers: 1<=n<=capacity)
// TODO use maxsparsity whenever we have a way of generating sparse rand data
int capacity = InfrastructureAnalyzer.getRemoteParallelMapTasks();
long dfsblocksize = InfrastructureAnalyzer.getHDFSBlockSize();
// correction max number of mappers on yarn clusters
if (InfrastructureAnalyzer.isYarnEnabled())
capacity = (int) Math.max(capacity, YarnClusterAnalyzer.getNumCores());
int nmapers = Math.max(Math.min((int) (8 * maxbrlen * maxbclen * (long) numblocks / dfsblocksize), capacity), 1);
job.setNumMapTasks(nmapers);
// set up what matrices are needed to pass from the mapper to reducer
HashSet<Byte> mapoutputIndexes = MRJobConfiguration.setUpOutputIndexesForMapper(job, realIndexes, dataGenInsStr, instructionsInMapper, null, aggInstructionsInReducer, otherInstructionsInReducer, resultIndexes);
MatrixChar_N_ReducerGroups ret = MRJobConfiguration.computeMatrixCharacteristics(job, realIndexes, dataGenInsStr, instructionsInMapper, null, aggInstructionsInReducer, null, otherInstructionsInReducer, resultIndexes, mapoutputIndexes, false);
stats = ret.stats;
// set up the number of reducers
MRJobConfiguration.setNumReducers(job, ret.numReducerGroups, numReducers);
// print the complete MRJob instruction
if (LOG.isTraceEnabled())
inst.printCompleteMRJobInstruction(stats);
// Update resultDimsUnknown based on computed "stats"
byte[] resultDimsUnknown = new byte[resultIndexes.length];
for (int i = 0; i < resultIndexes.length; i++) {
if (stats[i].getRows() == -1 || stats[i].getCols() == -1) {
resultDimsUnknown[i] = (byte) 1;
} else {
resultDimsUnknown[i] = (byte) 0;
}
}
boolean mayContainCtable = instructionsInMapper.contains("ctabletransform") || instructionsInMapper.contains("groupedagg");
// set up the multiple output files, and their format information
MRJobConfiguration.setUpMultipleOutputs(job, resultIndexes, resultDimsUnknown, outputs, outputInfos, true, mayContainCtable);
// configure mapper and the mapper output key value pairs
job.setMapperClass(DataGenMapper.class);
if (numReducers == 0) {
job.setMapOutputKeyClass(Writable.class);
job.setMapOutputValueClass(Writable.class);
} else {
job.setMapOutputKeyClass(MatrixIndexes.class);
job.setMapOutputValueClass(TaggedMatrixBlock.class);
}
// set up combiner
if (numReducers != 0 && aggInstructionsInReducer != null && !aggInstructionsInReducer.isEmpty())
job.setCombinerClass(GMRCombiner.class);
// configure reducer
job.setReducerClass(GMRReducer.class);
// job.setReducerClass(PassThroughReducer.class);
// By default, the job executes in "cluster" mode.
// Determine if we can optimize and run it in "local" mode.
MatrixCharacteristics[] inputStats = new MatrixCharacteristics[inputs.length];
for (int i = 0; i < inputs.length; i++) {
inputStats[i] = new MatrixCharacteristics(rlens[i], clens[i], brlens[i], bclens[i]);
}
// set unique working dir
MRJobConfiguration.setUniqueWorkingDir(job);
runjob = JobClient.runJob(job);
/* Process different counters */
Group group = runjob.getCounters().getGroup(MRJobConfiguration.NUM_NONZERO_CELLS);
for (int i = 0; i < resultIndexes.length; i++) {
// number of non-zeros
stats[i].setNonZeros(group.getCounter(Integer.toString(i)));
}
String dir = dimsUnknownFilePrefix + "/" + runjob.getID().toString() + "_dimsFile";
stats = MapReduceTool.processDimsFiles(dir, stats);
MapReduceTool.deleteFileIfExistOnHDFS(dir);
} finally {
for (String input : inputs) MapReduceTool.deleteFileIfExistOnHDFS(new Path(input), job);
}
return new JobReturn(stats, outputInfos, runjob.isSuccessful());
}
Aggregations