Example 1 with MRINSTRUCTION_TYPE
use of org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE in project incubator-systemml by apache.
the class CostEstimatorStaticRuntime method getNFLOP.
private double getNFLOP(String optype, boolean inMR, long d1m, long d1n, double d1s, long d2m, long d2n, double d2s, long d3m, long d3n, double d3s, String[] args) throws DMLRuntimeException {
//operation costs in FLOP on matrix block level (for CP and MR instructions)
//(excludes IO and parallelism; assumes known dims for all inputs, outputs )
boolean leftSparse = MatrixBlock.evalSparseFormatInMemory(d1m, d1n, (long) (d1s * d1m * d1n));
boolean rightSparse = MatrixBlock.evalSparseFormatInMemory(d2m, d2n, (long) (d2s * d2m * d2n));
boolean onlyLeft = (d1m >= 0 && d1n >= 0 && d2m < 0 && d2n < 0);
boolean allExists = (d1m >= 0 && d1n >= 0 && d2m >= 0 && d2n >= 0 && d3m >= 0 && d3n >= 0);
//NOTE: all instruction types that are equivalent in CP and MR are only
//included in CP to prevent redundancy
CPINSTRUCTION_TYPE cptype = CPInstructionParser.String2CPInstructionType.get(optype);
if (//for CP Ops and equivalent MR ops
cptype != null) {
//general approach: count of floating point *, /, +, -, ^, builtin ;
switch(cptype) {
case //opcodes: ba+*, cov
AggregateBinary:
if (optype.equals("ba+*")) {
//average flop count
if (!leftSparse && !rightSparse)
return 2 * (d1m * d1n * ((d2n > 1) ? d1s : 1.0) * d2n) / 2;
else if (!leftSparse && rightSparse)
return 2 * (d1m * d1n * d1s * d2n * d2s) / 2;
else if (leftSparse && !rightSparse)
return 2 * (d1m * d1n * d1s * d2n) / 2;
else
//leftSparse && rightSparse
return 2 * (d1m * d1n * d1s * d2n * d2s) / 2;
} else if (optype.equals("cov")) {
//(11+3*k+)
return 23 * d1m;
}
return 0;
case MMChain:
//(mmchain essentially two matrix-vector muliplications)
if (!leftSparse)
return (2 + 2) * (d1m * d1n) / 2;
else
return (2 + 2) * (d1m * d1n * d1s) / 2;
case //opcodes: tak+*
AggregateTernary:
//2*1(*) + 4 (k+)
return 6 * d1m * d1n;
case //opcodes: uak+, uark+, uack+, uasqk+, uarsqk+, uacsqk+,
AggregateUnary:
if (optype.equals("nrow") || optype.equals("ncol") || optype.equals("length"))
return DEFAULT_NFLOP_NOOP;
else if (optype.equals("cm")) {
double xcm = 1;
switch(Integer.parseInt(args[0])) {
//count
case 0:
xcm = 1;
break;
//mean
case 1:
xcm = 8;
break;
//cm2
case 2:
xcm = 16;
break;
//cm3
case 3:
xcm = 31;
break;
//cm4
case 4:
xcm = 51;
break;
//variance
case 5:
xcm = 16;
break;
}
return (leftSparse) ? xcm * (d1m * d1s + 1) : xcm * d1m;
} else if (optype.equals("uatrace") || optype.equals("uaktrace"))
return 2 * d1m * d1n;
else if (optype.equals("ua+") || optype.equals("uar+") || optype.equals("uac+")) {
//sparse safe operations
if (//dense
!leftSparse)
return d1m * d1n;
else
//sparse
return d1m * d1n * d1s;
} else if (optype.equals("uak+") || optype.equals("uark+") || optype.equals("uack+"))
//1*k+
return 4 * d1m * d1n;
else if (optype.equals("uasqk+") || optype.equals("uarsqk+") || optype.equals("uacsqk+"))
// +1 for multiplication to square term
return 5 * d1m * d1n;
else if (optype.equals("uamean") || optype.equals("uarmean") || optype.equals("uacmean"))
//1*k+
return 7 * d1m * d1n;
else if (optype.equals("uavar") || optype.equals("uarvar") || optype.equals("uacvar"))
return 14 * d1m * d1n;
else if (optype.equals("uamax") || optype.equals("uarmax") || optype.equals("uacmax") || optype.equals("uamin") || optype.equals("uarmin") || optype.equals("uacmin") || optype.equals("uarimax") || optype.equals("ua*"))
return d1m * d1n;
return 0;
case //opcodes: +, -, *, /, ^ (incl. ^2, *2)
ArithmeticBinary:
//note: covers scalar-scalar, scalar-matrix, matrix-matrix
if (optype.equals("+") || //sparse safe
optype.equals("-") && (leftSparse || rightSparse))
return d1m * d1n * d1s + d2m * d2n * d2s;
else
return d3m * d3n;
case //opcodes: ctable
Ternary:
if (optype.equals("ctable")) {
if (leftSparse)
//add
return d1m * d1n * d1s;
else
return d1m * d1n;
}
return 0;
case //opcodes: &&, ||
BooleanBinary:
//always scalar-scalar
return 1;
case //opcodes: !
BooleanUnary:
//always scalar-scalar
return 1;
case //opcodes: log
Builtin:
//note: can be unary or binary
if (//binary
allExists)
return 3 * d3m * d3n;
else
//unary
return d3m * d3n;
case //opcodes: max, min, solve
BuiltinBinary:
//note: covers scalar-scalar, scalar-matrix, matrix-matrix
if (//see also MultiReturnBuiltin
optype.equals("solve"))
//for 1kx1k ~ 1GFLOP -> 0.5s
return d1m * d1n * d1n;
else
//default
return d3m * d3n;
case //opcodes: exp, abs, sin, cos, tan, sign, sqrt, plogp, print, round, sprop, sigmoid
BuiltinUnary:
//TODO add cost functions for commons math builtins: inverse, cholesky
if (//scalar only
optype.equals("print"))
return 1;
else {
//default for all ops
double xbu = 1;
if (optype.equals("plogp"))
xbu = 2;
else if (optype.equals("round"))
xbu = 4;
if (optype.equals("sin") || optype.equals("tan") || optype.equals("round") || optype.equals("abs") || optype.equals("sqrt") || optype.equals("sprop") || optype.equals("sigmoid") || //sparse-safe
optype.equals("sign")) {
if (//sparse
leftSparse)
return xbu * d1m * d1n * d1s;
else
//dense
return xbu * d1m * d1n;
} else
return xbu * d1m * d1n;
}
//opcodes: r', rdiag
case Reorg:
case //opcodes: rshape
MatrixReshape:
if (leftSparse)
return d1m * d1n * d1s;
else
return d1m * d1n;
case //opcodes: append
Append:
return DEFAULT_NFLOP_CP * (((leftSparse) ? d1m * d1n * d1s : d1m * d1n) + ((rightSparse) ? d2m * d2n * d2s : d2m * d2n));
case //opcodes: ==, !=, <, >, <=, >=
RelationalBinary:
//note: all relational ops are not sparsesafe
return //covers all combinations of scalar and matrix
d3m * d3n;
case //opcodes: rm, mv
File:
return DEFAULT_NFLOP_NOOP;
case //opcodes: assignvar, cpvar, rmvar, rmfilevar, assignvarwithfile, attachfiletovar, valuepick, iqsize, read, write, createvar, setfilename, castAsMatrix
Variable:
if (optype.equals("write")) {
boolean text = args[0].equals("textcell") || args[0].equals("csv");
double xwrite = text ? DEFAULT_NFLOP_TEXT_IO : DEFAULT_NFLOP_CP;
if (!leftSparse)
return d1m * d1n * xwrite;
else
return d1m * d1n * d1s * xwrite;
} else if (optype.equals("inmem-iqm"))
//note: assumes uniform distribution
return //sum of weights
2 * d1m + 5 + //scan to lower quantile
0.25d * d1m + //scan from lower to upper quantile
8 * 0.5 * d1m;
else
return DEFAULT_NFLOP_NOOP;
case //opcodes: rand, seq
Rand:
if (optype.equals(DataGen.RAND_OPCODE)) {
//per random number
int nflopRand = 32;
switch(Integer.parseInt(args[0])) {
//empty matrix
case 0:
return DEFAULT_NFLOP_NOOP;
//allocate, arrayfill
case 1:
return d3m * d3n * 8;
case //full rand
2:
{
if (d3s == 1.0)
//DENSE gen (incl allocate)
return d3m * d3n * nflopRand + d3m * d3n * 8;
else
return (d3s >= MatrixBlock.SPARSITY_TURN_POINT) ? //DENSE gen (incl allocate)
2 * d3m * d3n * nflopRand + d3m * d3n * 8 : //SPARSE gen (incl allocate)
3 * d3m * d3n * d3s * nflopRand + d3m * d3n * d3s * 24;
}
}
} else
//seq
return d3m * d3n * DEFAULT_NFLOP_CP;
case //sinit
StringInit:
return d3m * d3n * DEFAULT_NFLOP_CP;
case //opcodes: extfunct
External:
//note: should be invoked independently for multiple outputs
return d1m * d1n * d1s * DEFAULT_NFLOP_UNKNOWN;
case //opcodes: qr, lu, eigen
MultiReturnBuiltin:
//note: they all have cubic complexity, the scaling factor refers to commons.math
//default e.g, qr
double xf = 2;
if (optype.equals("eigen"))
xf = 32;
else if (optype.equals("lu"))
xf = 16;
//for 1kx1k ~ 2GFLOP -> 1s
return xf * d1m * d1n * d1n;
case //opcodes: cdf, invcdf, groupedagg, rmempty
ParameterizedBuiltin:
if (optype.equals("cdf") || optype.equals("invcdf"))
//scalar call to commons.math
return DEFAULT_NFLOP_UNKNOWN;
else if (optype.equals("groupedagg")) {
double xga = 1;
switch(Integer.parseInt(args[0])) {
//sum, see uk+
case 0:
xga = 4;
break;
//count, see cm
case 1:
xga = 1;
break;
//mean
case 2:
xga = 8;
break;
//cm2
case 3:
xga = 16;
break;
//cm3
case 4:
xga = 31;
break;
//cm4
case 5:
xga = 51;
break;
//variance
case 6:
xga = 16;
break;
}
//scan for min/max, groupedagg
return 2 * d1m + xga * d1m;
} else if (optype.equals("rmempty")) {
switch(Integer.parseInt(args[0])) {
case //remove rows
0:
return ((leftSparse) ? d1m : d1m * Math.ceil(1.0d / d1s) / 2) + DEFAULT_NFLOP_CP * d3m * d2m;
case //remove cols
1:
return d1n * Math.ceil(1.0d / d1s) / 2 + DEFAULT_NFLOP_CP * d3m * d2m;
}
}
return 0;
case //opcodes: sort
QSort:
if (optype.equals("sort")) {
//note: mergesort since comparator used
double sortCosts = 0;
if (onlyLeft)
sortCosts = DEFAULT_NFLOP_CP * d1m + d1m;
else
//w/ weights
sortCosts = DEFAULT_NFLOP_CP * ((leftSparse) ? d1m * d1s : d1m);
return //mergesort
sortCosts + d1m * (int) (Math.log(d1m) / Math.log(2)) + DEFAULT_NFLOP_CP * d1m;
}
return 0;
case //opcodes: rangeReIndex, leftIndex
MatrixIndexing:
if (optype.equals("leftIndex")) {
return DEFAULT_NFLOP_CP * ((leftSparse) ? d1m * d1n * d1s : d1m * d1n) + 2 * DEFAULT_NFLOP_CP * ((rightSparse) ? d2m * d2n * d2s : d2m * d2n);
} else if (optype.equals("rangeReIndex")) {
return DEFAULT_NFLOP_CP * ((leftSparse) ? d2m * d2n * d2s : d2m * d2n);
}
return 0;
case //opcodes: tsmm
MMTSJ:
//average flop count
if (MMTSJType.valueOf(args[0]).isLeft()) {
//lefttranspose
if (//dense
!rightSparse)
return d1m * d1n * d1s * d1n / 2;
else
//sparse
return d1m * d1n * d1s * d1n * d1s / 2;
} else if (onlyLeft) {
//righttranspose
if (//dense
!leftSparse)
return (double) d1m * d1n * d1m / 2;
else
//sparse
return //reorg sparse
d1m * d1n * d1s + //core tsmm
d1m * d1n * d1s * d1n * d1s / 2;
}
return 0;
case Partition:
return //partitioning costs
d1m * d1n * d1s + (//include write cost if in CP
inMR ? //include write cost if in CP
0 : getHDFSWriteTime(d1m, d1n, d1s) * DEFAULT_FLOPS);
case INVALID:
return 0;
default:
throw new DMLRuntimeException("CostEstimator: unsupported instruction type: " + optype);
}
}
//if not found in CP instructions
MRINSTRUCTION_TYPE mrtype = MRInstructionParser.String2MRInstructionType.get(optype);
if (//for specific MR ops
mrtype != null) {
switch(mrtype) {
case //opcodes: a+, ak+, asqk+, a*, amax, amin, amean
Aggregate:
//TODO should be aggregate unary
int numMap = Integer.parseInt(args[0]);
if (optype.equals("ak+"))
return 4 * numMap * d1m * d1n * d1s;
else if (optype.equals("asqk+"))
// +1 for multiplication to square term
return 5 * numMap * d1m * d1n * d1s;
else if (optype.equals("avar"))
return 14 * numMap * d1m * d1n * d1s;
else
return numMap * d1m * d1n * d1s;
case //opcodes: cpmm, rmm, mapmult
AggregateBinary:
//note: copy from CP costs
if (optype.equals("cpmm") || optype.equals("rmm") || //matrix mult
optype.equals(MapMult.OPCODE)) {
//average flop count
if (!leftSparse && !rightSparse)
return 2 * (d1m * d1n * ((d2n > 1) ? d1s : 1.0) * d2n) / 2;
else if (!leftSparse && rightSparse)
return 2 * (d1m * d1n * d1s * d2n * d2s) / 2;
else if (leftSparse && !rightSparse)
return 2 * (d1m * d1n * d1s * d2n) / 2;
else
//leftSparse && rightSparse
return 2 * (d1m * d1n * d1s * d2n * d2s) / 2;
}
return 0;
case //opcodes: mapmultchain
MapMultChain:
//assume dense input2 and input3
return //ba(+*)
2 * d1m * d2n * d1n * ((d2n > 1) ? d1s : 1.0) + //cellwise b(*)
d1m * d2n + //r(t)
d1m * d2n + //ba(+*)
2 * d2n * d1n * d1m * (leftSparse ? d1s : 1.0) + //r(t)
d2n * d1n;
case //opcodes: s-r, so, max, min,
ArithmeticBinary:
//note: all relational ops are not sparsesafe
return //covers all combinations of scalar and matrix
d3m * d3n;
case //opcodes: combineunary
CombineUnary:
return d1m * d1n * d1s;
case //opcodes: combinebinary
CombineBinary:
return d1m * d1n * d1s + d2m * d2n * d2s;
case //opcodes: combinetertiary
CombineTernary:
return d1m * d1n * d1s + d2m * d2n * d2s + d3m * d3n * d3s;
case //opcodes: log, slog, pow
Unary:
//note: covers scalar, matrix, matrix-scalar
return d3m * d3n;
case //opcodes: ctabletransform, ctabletransformscalarweight, ctabletransformhistogram, ctabletransformweightedhistogram
Ternary:
//note: copy from cp
if (leftSparse)
//add
return d1m * d1n * d1s;
else
return d1m * d1n;
case Quaternary:
//TODO pattern specific and all 4 inputs requires
return d1m * d1n * d1s * 4;
case //opcodes: rblk
Reblock:
return DEFAULT_NFLOP_CP * ((leftSparse) ? d1m * d1n * d1s : d1m * d1n);
case //opcodes: rblk
Replicate:
return DEFAULT_NFLOP_CP * ((leftSparse) ? d1m * d1n * d1s : d1m * d1n);
case //opcodes: mean
CM_N_COV:
double xcm = 8;
return (leftSparse) ? xcm * (d1m * d1s + 1) : xcm * d1m;
case //opcodes: groupedagg
GroupedAggregate:
//TODO: need to consolidate categories (ParameterizedBuiltin)
//copy from CP opertion
double xga = 1;
switch(Integer.parseInt(args[0])) {
//sum, see uk+
case 0:
xga = 4;
break;
//count, see cm
case 1:
xga = 1;
break;
//mean
case 2:
xga = 8;
break;
//cm2
case 3:
xga = 16;
break;
//cm3
case 4:
xga = 31;
break;
//cm4
case 5:
xga = 51;
break;
//variance
case 6:
xga = 16;
break;
}
//scan for min/max, groupedagg
return 2 * d1m + xga * d1m;
case //opcodes: valuepick, rangepick
PickByCount:
break;
case //opcodes: rangeReIndex, rangeReIndexForLeft
RangeReIndex:
//TODO: requires category consolidation
if (optype.equals("rangeReIndex"))
return DEFAULT_NFLOP_CP * ((leftSparse) ? d2m * d2n * d2s : d2m * d2n);
else
//rangeReIndexForLeft
return DEFAULT_NFLOP_CP * ((leftSparse) ? d1m * d1n * d1s : d1m * d1n) + DEFAULT_NFLOP_CP * ((rightSparse) ? d2m * d2n * d2s : d2m * d2n);
case //opcodes: zeroOut
ZeroOut:
return DEFAULT_NFLOP_CP * ((leftSparse) ? d1m * d1n * d1s : d1m * d1n) + DEFAULT_NFLOP_CP * ((rightSparse) ? d2m * d2n * d2s : d2m * d2n);
default:
return 0;
}
} else {
throw new DMLRuntimeException("CostEstimator: unsupported instruction type: " + optype);
}
//should never come here.
return -1;
}
Also used :
MRINSTRUCTION_TYPE(org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE)
CPINSTRUCTION_TYPE(org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE)
DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException)
Example 2 with MRINSTRUCTION_TYPE
use of org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE in project incubator-systemml by apache.
the class MRInstructionParser method parseCombineInstructions.
public static MRInstruction[] parseCombineInstructions(String str) throws DMLRuntimeException {
MRInstruction[] inst = null;
if (str != null && !str.isEmpty()) {
String[] strlist = str.split(Instruction.INSTRUCTION_DELIM);
inst = new MRInstruction[strlist.length];
for (int i = 0; i < strlist.length; i++) {
MRINSTRUCTION_TYPE type = InstructionUtils.getMRType(strlist[i]);
if (type == MRINSTRUCTION_TYPE.CombineBinary)
inst[i] = (CombineBinaryInstruction) CombineBinaryInstruction.parseInstruction(strlist[i]);
else if (type == MRINSTRUCTION_TYPE.CombineTernary)
inst[i] = (CombineTernaryInstruction) CombineTernaryInstruction.parseInstruction(strlist[i]);
else
throw new DMLRuntimeException("unknown combine instruction: " + strlist[i]);
}
}
return inst;
}
Also used :
MRINSTRUCTION_TYPE(org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE)
ParameterizedBuiltinMRInstruction(org.apache.sysml.runtime.instructions.mr.ParameterizedBuiltinMRInstruction)
PMMJMRInstruction(org.apache.sysml.runtime.instructions.mr.PMMJMRInstruction)
DataGenMRInstruction(org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction)
MMTSJMRInstruction(org.apache.sysml.runtime.instructions.mr.MMTSJMRInstruction)
DataPartitionMRInstruction(org.apache.sysml.runtime.instructions.mr.DataPartitionMRInstruction)
MatrixReshapeMRInstruction(org.apache.sysml.runtime.instructions.mr.MatrixReshapeMRInstruction)
MRInstruction(org.apache.sysml.runtime.instructions.mr.MRInstruction)
RemoveEmptyMRInstruction(org.apache.sysml.runtime.instructions.mr.RemoveEmptyMRInstruction)
CombineBinaryInstruction(org.apache.sysml.runtime.instructions.mr.CombineBinaryInstruction)
DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException)
Example 3 with MRINSTRUCTION_TYPE
use of org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE in project incubator-systemml by apache.
the class DataGenMR method runJob.
/**
* <p>Starts a Rand MapReduce job which will produce one or more random objects.</p>
*
* @param inst MR job instruction
* @param dataGenInstructions array of data gen instructions
* @param instructionsInMapper instructions in mapper
* @param aggInstructionsInReducer aggregate instructions in reducer
* @param otherInstructionsInReducer other instructions in reducer
* @param numReducers number of reducers
* @param replication file replication
* @param resultIndexes result indexes for each random object
* @param dimsUnknownFilePrefix file path prefix when dimensions unknown
* @param outputs output file for each random object
* @param outputInfos output information for each random object
* @return matrix characteristics for each random object
* @throws Exception if Exception occurs
*/
public static JobReturn runJob(MRJobInstruction inst, String[] dataGenInstructions, String instructionsInMapper, String aggInstructionsInReducer, String otherInstructionsInReducer, int numReducers, int replication, byte[] resultIndexes, String dimsUnknownFilePrefix, String[] outputs, OutputInfo[] outputInfos) throws Exception {
JobConf job = new JobConf(DataGenMR.class);
job.setJobName("DataGen-MR");
//whether use block representation or cell representation
MRJobConfiguration.setMatrixValueClass(job, true);
byte[] realIndexes = new byte[dataGenInstructions.length];
for (byte b = 0; b < realIndexes.length; b++) realIndexes[b] = b;
String[] inputs = new String[dataGenInstructions.length];
InputInfo[] inputInfos = new InputInfo[dataGenInstructions.length];
long[] rlens = new long[dataGenInstructions.length];
long[] clens = new long[dataGenInstructions.length];
int[] brlens = new int[dataGenInstructions.length];
int[] bclens = new int[dataGenInstructions.length];
FileSystem fs = FileSystem.get(job);
String dataGenInsStr = "";
int numblocks = 0;
int maxbrlen = -1, maxbclen = -1;
double maxsparsity = -1;
for (int i = 0; i < dataGenInstructions.length; i++) {
dataGenInsStr = dataGenInsStr + Lop.INSTRUCTION_DELIMITOR + dataGenInstructions[i];
MRInstruction mrins = MRInstructionParser.parseSingleInstruction(dataGenInstructions[i]);
MRINSTRUCTION_TYPE mrtype = mrins.getMRInstructionType();
DataGenMRInstruction genInst = (DataGenMRInstruction) mrins;
rlens[i] = genInst.getRows();
clens[i] = genInst.getCols();
brlens[i] = genInst.getRowsInBlock();
bclens[i] = genInst.getColsInBlock();
maxbrlen = Math.max(maxbrlen, brlens[i]);
maxbclen = Math.max(maxbclen, bclens[i]);
if (mrtype == MRINSTRUCTION_TYPE.Rand) {
RandInstruction randInst = (RandInstruction) mrins;
inputs[i] = LibMatrixDatagen.generateUniqueSeedPath(genInst.getBaseDir());
maxsparsity = Math.max(maxsparsity, randInst.getSparsity());
PrintWriter pw = null;
try {
pw = new PrintWriter(fs.create(new Path(inputs[i])));
//for obj reuse and preventing repeated buffer re-allocations
StringBuilder sb = new StringBuilder();
//seed generation
Well1024a bigrand = LibMatrixDatagen.setupSeedsForRand(randInst.getSeed());
LongStream nnz = LibMatrixDatagen.computeNNZperBlock(rlens[i], clens[i], brlens[i], bclens[i], randInst.getSparsity());
PrimitiveIterator.OfLong nnzIter = nnz.iterator();
for (long r = 0; r < rlens[i]; r += brlens[i]) {
long curBlockRowSize = Math.min(brlens[i], (rlens[i] - r));
for (long c = 0; c < clens[i]; c += bclens[i]) {
long curBlockColSize = Math.min(bclens[i], (clens[i] - c));
sb.append((r / brlens[i]) + 1);
sb.append(',');
sb.append((c / bclens[i]) + 1);
sb.append(',');
sb.append(curBlockRowSize);
sb.append(',');
sb.append(curBlockColSize);
sb.append(',');
sb.append(nnzIter.nextLong());
sb.append(',');
sb.append(bigrand.nextLong());
pw.println(sb.toString());
sb.setLength(0);
numblocks++;
}
}
} finally {
IOUtilFunctions.closeSilently(pw);
}
inputInfos[i] = InputInfo.TextCellInputInfo;
} else if (mrtype == MRINSTRUCTION_TYPE.Seq) {
SeqInstruction seqInst = (SeqInstruction) mrins;
inputs[i] = genInst.getBaseDir() + System.currentTimeMillis() + ".seqinput";
//always dense
maxsparsity = 1.0;
double from = seqInst.fromValue;
double to = seqInst.toValue;
double incr = seqInst.incrValue;
//handle default 1 to -1 for special case of from>to
incr = LibMatrixDatagen.updateSeqIncr(from, to, incr);
// Correctness checks on (from, to, incr)
boolean neg = (from > to);
if (incr == 0)
throw new DMLRuntimeException("Invalid value for \"increment\" in seq().");
if (neg != (incr < 0))
throw new DMLRuntimeException("Wrong sign for the increment in a call to seq()");
// Compute the number of rows in the sequence
long numrows = 1 + (long) Math.floor((to - from) / incr);
if (rlens[i] > 0) {
if (numrows != rlens[i])
throw new DMLRuntimeException("Unexpected error while processing sequence instruction. Expected number of rows does not match given number: " + rlens[i] + " != " + numrows);
} else {
rlens[i] = numrows;
}
if (clens[i] > 0 && clens[i] != 1)
throw new DMLRuntimeException("Unexpected error while processing sequence instruction. Number of columns (" + clens[i] + ") must be equal to 1.");
else
clens[i] = 1;
PrintWriter pw = null;
try {
pw = new PrintWriter(fs.create(new Path(inputs[i])));
StringBuilder sb = new StringBuilder();
double temp = from;
double block_from, block_to;
for (long r = 0; r < rlens[i]; r += brlens[i]) {
long curBlockRowSize = Math.min(brlens[i], (rlens[i] - r));
// block (bid_i,bid_j) generates a sequence from the interval [block_from, block_to] (inclusive of both end points of the interval)
long bid_i = ((r / brlens[i]) + 1);
long bid_j = 1;
block_from = temp;
block_to = temp + (curBlockRowSize - 1) * incr;
// next block starts from here
temp = block_to + incr;
sb.append(bid_i);
sb.append(',');
sb.append(bid_j);
sb.append(',');
sb.append(block_from);
sb.append(',');
sb.append(block_to);
sb.append(',');
sb.append(incr);
pw.println(sb.toString());
sb.setLength(0);
numblocks++;
}
} finally {
IOUtilFunctions.closeSilently(pw);
}
inputInfos[i] = InputInfo.TextCellInputInfo;
} else {
throw new DMLRuntimeException("Unexpected Data Generation Instruction Type: " + mrtype);
}
}
//remove the first ","
dataGenInsStr = dataGenInsStr.substring(1);
RunningJob runjob;
MatrixCharacteristics[] stats;
try {
//set up the block size
MRJobConfiguration.setBlocksSizes(job, realIndexes, brlens, bclens);
//set up the input files and their format information
MRJobConfiguration.setUpMultipleInputs(job, realIndexes, inputs, inputInfos, brlens, bclens, false, ConvertTarget.BLOCK);
//set up the dimensions of input matrices
MRJobConfiguration.setMatricesDimensions(job, realIndexes, rlens, clens);
MRJobConfiguration.setDimsUnknownFilePrefix(job, dimsUnknownFilePrefix);
//set up the block size
MRJobConfiguration.setBlocksSizes(job, realIndexes, brlens, bclens);
//set up the rand Instructions
MRJobConfiguration.setRandInstructions(job, dataGenInsStr);
//set up unary instructions that will perform in the mapper
MRJobConfiguration.setInstructionsInMapper(job, instructionsInMapper);
//set up the aggregate instructions that will happen in the combiner and reducer
MRJobConfiguration.setAggregateInstructions(job, aggInstructionsInReducer);
//set up the instructions that will happen in the reducer, after the aggregation instrucions
MRJobConfiguration.setInstructionsInReducer(job, otherInstructionsInReducer);
//set up the replication factor for the results
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up map/reduce memory configurations (if in AM context)
DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//determine degree of parallelism (nmappers: 1<=n<=capacity)
//TODO use maxsparsity whenever we have a way of generating sparse rand data
int capacity = InfrastructureAnalyzer.getRemoteParallelMapTasks();
long dfsblocksize = InfrastructureAnalyzer.getHDFSBlockSize();
//correction max number of mappers on yarn clusters
if (InfrastructureAnalyzer.isYarnEnabled())
capacity = (int) Math.max(capacity, YarnClusterAnalyzer.getNumCores());
int nmapers = Math.max(Math.min((int) (8 * maxbrlen * maxbclen * (long) numblocks / dfsblocksize), capacity), 1);
job.setNumMapTasks(nmapers);
//set up what matrices are needed to pass from the mapper to reducer
HashSet<Byte> mapoutputIndexes = MRJobConfiguration.setUpOutputIndexesForMapper(job, realIndexes, dataGenInsStr, instructionsInMapper, null, aggInstructionsInReducer, otherInstructionsInReducer, resultIndexes);
MatrixChar_N_ReducerGroups ret = MRJobConfiguration.computeMatrixCharacteristics(job, realIndexes, dataGenInsStr, instructionsInMapper, null, aggInstructionsInReducer, null, otherInstructionsInReducer, resultIndexes, mapoutputIndexes, false);
stats = ret.stats;
//set up the number of reducers
MRJobConfiguration.setNumReducers(job, ret.numReducerGroups, numReducers);
// print the complete MRJob instruction
if (LOG.isTraceEnabled())
inst.printCompleteMRJobInstruction(stats);
// Update resultDimsUnknown based on computed "stats"
byte[] resultDimsUnknown = new byte[resultIndexes.length];
for (int i = 0; i < resultIndexes.length; i++) {
if (stats[i].getRows() == -1 || stats[i].getCols() == -1) {
resultDimsUnknown[i] = (byte) 1;
} else {
resultDimsUnknown[i] = (byte) 0;
}
}
boolean mayContainCtable = instructionsInMapper.contains("ctabletransform") || instructionsInMapper.contains("groupedagg");
//set up the multiple output files, and their format information
MRJobConfiguration.setUpMultipleOutputs(job, resultIndexes, resultDimsUnknown, outputs, outputInfos, true, mayContainCtable);
// configure mapper and the mapper output key value pairs
job.setMapperClass(DataGenMapper.class);
if (numReducers == 0) {
job.setMapOutputKeyClass(Writable.class);
job.setMapOutputValueClass(Writable.class);
} else {
job.setMapOutputKeyClass(MatrixIndexes.class);
job.setMapOutputValueClass(TaggedMatrixBlock.class);
}
//set up combiner
if (numReducers != 0 && aggInstructionsInReducer != null && !aggInstructionsInReducer.isEmpty())
job.setCombinerClass(GMRCombiner.class);
//configure reducer
job.setReducerClass(GMRReducer.class);
//job.setReducerClass(PassThroughReducer.class);
// By default, the job executes in "cluster" mode.
// Determine if we can optimize and run it in "local" mode.
MatrixCharacteristics[] inputStats = new MatrixCharacteristics[inputs.length];
for (int i = 0; i < inputs.length; i++) {
inputStats[i] = new MatrixCharacteristics(rlens[i], clens[i], brlens[i], bclens[i]);
}
//set unique working dir
MRJobConfiguration.setUniqueWorkingDir(job);
runjob = JobClient.runJob(job);
/* Process different counters */
Group group = runjob.getCounters().getGroup(MRJobConfiguration.NUM_NONZERO_CELLS);
for (int i = 0; i < resultIndexes.length; i++) {
// number of non-zeros
stats[i].setNonZeros(group.getCounter(Integer.toString(i)));
}
String dir = dimsUnknownFilePrefix + "/" + runjob.getID().toString() + "_dimsFile";
stats = MapReduceTool.processDimsFiles(dir, stats);
MapReduceTool.deleteFileIfExistOnHDFS(dir);
} finally {
for (String input : inputs) MapReduceTool.deleteFileIfExistOnHDFS(new Path(input), job);
}
return new JobReturn(stats, outputInfos, runjob.isSuccessful());
}
Also used :
Group(org.apache.hadoop.mapred.Counters.Group)
DataGenMRInstruction(org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction)
InputInfo(org.apache.sysml.runtime.matrix.data.InputInfo)
GMRCombiner(org.apache.sysml.runtime.matrix.mapred.GMRCombiner)
FileSystem(org.apache.hadoop.fs.FileSystem)
DataGenMRInstruction(org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction)
MRInstruction(org.apache.sysml.runtime.instructions.mr.MRInstruction)
JobConf(org.apache.hadoop.mapred.JobConf)
PrintWriter(java.io.PrintWriter)
Path(org.apache.hadoop.fs.Path)
DMLConfig(org.apache.sysml.conf.DMLConfig)
PrimitiveIterator(java.util.PrimitiveIterator)
SeqInstruction(org.apache.sysml.runtime.instructions.mr.SeqInstruction)
LongStream(java.util.stream.LongStream)
RandInstruction(org.apache.sysml.runtime.instructions.mr.RandInstruction)
DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException)
MatrixChar_N_ReducerGroups(org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.MatrixChar_N_ReducerGroups)
MRINSTRUCTION_TYPE(org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE)
RunningJob(org.apache.hadoop.mapred.RunningJob)
Well1024a(org.apache.commons.math3.random.Well1024a)
Aggregations
DMLRuntimeException (org.apache.sysml.runtime.DMLRuntimeException)3
MRINSTRUCTION_TYPE (org.apache.sysml.runtime.instructions.mr.MRInstruction.MRINSTRUCTION_TYPE)3
DataGenMRInstruction (org.apache.sysml.runtime.instructions.mr.DataGenMRInstruction)2
MRInstruction (org.apache.sysml.runtime.instructions.mr.MRInstruction)2
PrintWriter (java.io.PrintWriter)1
PrimitiveIterator (java.util.PrimitiveIterator)1
LongStream (java.util.stream.LongStream)1
Well1024a (org.apache.commons.math3.random.Well1024a)1
FileSystem (org.apache.hadoop.fs.FileSystem)1
Path (org.apache.hadoop.fs.Path)1
Group (org.apache.hadoop.mapred.Counters.Group)1
JobConf (org.apache.hadoop.mapred.JobConf)1
RunningJob (org.apache.hadoop.mapred.RunningJob)1
DMLConfig (org.apache.sysml.conf.DMLConfig)1
CPINSTRUCTION_TYPE (org.apache.sysml.runtime.instructions.cp.CPInstruction.CPINSTRUCTION_TYPE)1
CombineBinaryInstruction (org.apache.sysml.runtime.instructions.mr.CombineBinaryInstruction)1
DataPartitionMRInstruction (org.apache.sysml.runtime.instructions.mr.DataPartitionMRInstruction)1
MMTSJMRInstruction (org.apache.sysml.runtime.instructions.mr.MMTSJMRInstruction)1
MatrixReshapeMRInstruction (org.apache.sysml.runtime.instructions.mr.MatrixReshapeMRInstruction)1
PMMJMRInstruction (org.apache.sysml.runtime.instructions.mr.PMMJMRInstruction)1
