Examples with AggregateInstruction org.apache.sysml.runtime.instructions.mr.AggregateInstruction used on opensource projects

Example 6 with AggregateInstruction

use of org.apache.sysml.runtime.instructions.mr.AggregateInstruction in project incubator-systemml by apache.

the class ReduceBase method configure.

@Override
public void configure(JobConf job) {
    super.configure(job);
    reducerID = job.get(MRConfigurationNames.MR_TASK_ATTEMPT_ID);
    dimsUnknownFilePrefix = job.get("dims.unknown.file.prefix");
    // get the indexes of the final output matrices
    resultIndexes = MRJobConfiguration.getResultIndexes(job);
    resultDimsUnknown = MRJobConfiguration.getResultDimsUnknown(job);
    // initialize SystemML Counters (defined in MRJobConfiguration)
    resultsNonZeros = new long[resultIndexes.length];
    resultsMaxRowDims = new long[resultIndexes.length];
    resultsMaxColDims = new long[resultIndexes.length];
    collectFinalMultipleOutputs = MRJobConfiguration.getMultipleConvertedOutputs(job);
    // parse aggregate operations
    AggregateInstruction[] agg_insts = null;
    try {
        agg_insts = MRJobConfiguration.getAggregateInstructions(job);
        // parse unary and binary operations
        MRInstruction[] tmp = MRJobConfiguration.getInstructionsInReducer(job);
        if (tmp != null) {
            mixed_instructions = new ArrayList<>();
            Collections.addAll(mixed_instructions, tmp);
        }
    } catch (DMLRuntimeException e) {
        throw new RuntimeException(e);
    }
    // load data from distributed cache (if required, reuse if jvm_reuse)
    try {
        setupDistCacheFiles(job);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
    // reorganize the aggregate instructions, so that they are all associatied with each input
    if (agg_insts != null) {
        for (AggregateInstruction ins : agg_insts) {
            // associate instruction to its input
            ArrayList<AggregateInstruction> vec = agg_instructions.get(ins.input);
            if (vec == null) {
                vec = new ArrayList<>();
                agg_instructions.put(ins.input, vec);
            }
            vec.add(ins);
            if (ins.input == ins.output)
                continue;
            // need to add new aggregate instructions so that partial aggregation can be applied
            // this is important for combiner in the reducer side
            AggregateInstruction partialIns = new AggregateInstruction(ins.getOperator(), ins.output, ins.output, ins.toString());
            vec = agg_instructions.get(partialIns.input);
            if (vec == null) {
                vec = new ArrayList<>();
                agg_instructions.put(partialIns.input, vec);
            }
            vec.add(partialIns);
        }
    }
}

Example 7 with AggregateInstruction

use of org.apache.sysml.runtime.instructions.mr.AggregateInstruction in project systemml by apache.

the class MMCJMRCombinerReducerBase method performAggregateInstructions.

protected MatrixValue performAggregateInstructions(TaggedFirstSecondIndexes indexes, Iterator<MatrixValue> values) throws IOException {
    // manipulation on the tags first
    byte realTag = indexes.getTag();
    byte representTag;
    if (realTag == tagForLeft)
        representTag = aggBinInstruction.input1;
    else
        representTag = aggBinInstruction.input2;
    ArrayList<AggregateInstruction> instructions = agg_instructions.get(representTag);
    AggregateInstruction ins;
    if (instructions == null) {
        defaultAggIns.input = realTag;
        defaultAggIns.output = realTag;
        ins = defaultAggIns;
    } else {
        if (instructions.size() > 1)
            throw new IOException("only one aggregate operation on input " + indexes.getTag() + " is allowed in BlockMMCJMR");
        ins = instructions.get(0);
        if (ins.input != ins.output)
            throw new IOException("input index and output index have to be " + "the same for aggregate instructions in BlockMMCJMR");
    }
    // performa aggregation before doing mmcj
    // TODO: customize the code, since aggregation for matrix multiplcation can only be sum
    boolean needStartAgg = true;
    try {
        while (values.hasNext()) {
            MatrixValue value = values.next();
            if (needStartAgg) {
                buffer.reset(value.getNumRows(), value.getNumColumns(), value.isInSparseFormat());
                needStartAgg = false;
            // LOG.info("initialize buffer: sparse="+buffer.isInSparseFormat()+", nonZero="+buffer.getNonZeros());
            }
            buffer.binaryOperationsInPlace(((AggregateOperator) ins.getOperator()).increOp, value);
        // LOG.info("increment buffer: sparse="+buffer.isInSparseFormat()+", nonZero="+buffer.getNonZeros());
        }
    } catch (Exception e) {
        throw new IOException(e);
    }
    if (needStartAgg)
        return null;
    else
        return buffer;
}

Example 8 with AggregateInstruction

use of org.apache.sysml.runtime.instructions.mr.AggregateInstruction in project systemml by apache.

the class MatrixCharacteristics method computeDimension.

public static void computeDimension(HashMap<Byte, MatrixCharacteristics> dims, MRInstruction ins) {
    MatrixCharacteristics dimOut = dims.get(ins.output);
    if (dimOut == null) {
        dimOut = new MatrixCharacteristics();
        dims.put(ins.output, dimOut);
    }
    if (ins instanceof ReorgInstruction) {
        ReorgInstruction realIns = (ReorgInstruction) ins;
        reorg(dims.get(realIns.input), (ReorgOperator) realIns.getOperator(), dimOut);
    } else if (ins instanceof AppendInstruction) {
        AppendInstruction realIns = (AppendInstruction) ins;
        MatrixCharacteristics in_dim1 = dims.get(realIns.input1);
        MatrixCharacteristics in_dim2 = dims.get(realIns.input2);
        if (realIns.isCBind())
            dimOut.set(in_dim1.numRows, in_dim1.numColumns + in_dim2.numColumns, in_dim1.numRowsPerBlock, in_dim2.numColumnsPerBlock);
        else
            dimOut.set(in_dim1.numRows + in_dim2.numRows, in_dim1.numColumns, in_dim1.numRowsPerBlock, in_dim2.numColumnsPerBlock);
    } else if (ins instanceof CumulativeAggregateInstruction) {
        AggregateUnaryInstruction realIns = (AggregateUnaryInstruction) ins;
        MatrixCharacteristics in = dims.get(realIns.input);
        dimOut.set((long) Math.ceil((double) in.getRows() / in.getRowsPerBlock()), in.getCols(), in.getRowsPerBlock(), in.getColsPerBlock());
    } else if (ins instanceof AggregateUnaryInstruction) {
        AggregateUnaryInstruction realIns = (AggregateUnaryInstruction) ins;
        aggregateUnary(dims.get(realIns.input), (AggregateUnaryOperator) realIns.getOperator(), dimOut);
    } else if (ins instanceof AggregateBinaryInstruction) {
        AggregateBinaryInstruction realIns = (AggregateBinaryInstruction) ins;
        aggregateBinary(dims.get(realIns.input1), dims.get(realIns.input2), (AggregateBinaryOperator) realIns.getOperator(), dimOut);
    } else if (ins instanceof MapMultChainInstruction) {
        // output size independent of chain type
        MapMultChainInstruction realIns = (MapMultChainInstruction) ins;
        MatrixCharacteristics mc1 = dims.get(realIns.getInput1());
        MatrixCharacteristics mc2 = dims.get(realIns.getInput2());
        dimOut.set(mc1.numColumns, mc2.numColumns, mc1.numRowsPerBlock, mc1.numColumnsPerBlock);
    } else if (ins instanceof QuaternaryInstruction) {
        QuaternaryInstruction realIns = (QuaternaryInstruction) ins;
        MatrixCharacteristics mc1 = dims.get(realIns.getInput1());
        MatrixCharacteristics mc2 = dims.get(realIns.getInput2());
        MatrixCharacteristics mc3 = dims.get(realIns.getInput3());
        realIns.computeMatrixCharacteristics(mc1, mc2, mc3, dimOut);
    } else if (ins instanceof ReblockInstruction) {
        ReblockInstruction realIns = (ReblockInstruction) ins;
        MatrixCharacteristics in_dim = dims.get(realIns.input);
        dimOut.set(in_dim.numRows, in_dim.numColumns, realIns.brlen, realIns.bclen, in_dim.nonZero);
    } else if (ins instanceof MatrixReshapeMRInstruction) {
        MatrixReshapeMRInstruction mrinst = (MatrixReshapeMRInstruction) ins;
        MatrixCharacteristics in_dim = dims.get(mrinst.input);
        dimOut.set(mrinst.getNumRows(), mrinst.getNumColunms(), in_dim.getRowsPerBlock(), in_dim.getColsPerBlock(), in_dim.getNonZeros());
    } else if (ins instanceof RandInstruction || ins instanceof SeqInstruction) {
        DataGenMRInstruction dataIns = (DataGenMRInstruction) ins;
        dimOut.set(dims.get(dataIns.getInput()));
    } else if (ins instanceof ReplicateInstruction) {
        ReplicateInstruction realIns = (ReplicateInstruction) ins;
        realIns.computeOutputDimension(dims.get(realIns.input), dimOut);
    } else if (// before unary
    ins instanceof ParameterizedBuiltinMRInstruction) {
        ParameterizedBuiltinMRInstruction realIns = (ParameterizedBuiltinMRInstruction) ins;
        realIns.computeOutputCharacteristics(dims.get(realIns.input), dimOut);
    } else if (ins instanceof ScalarInstruction || ins instanceof AggregateInstruction || (ins instanceof UnaryInstruction && !(ins instanceof MMTSJMRInstruction)) || ins instanceof ZeroOutInstruction) {
        UnaryMRInstructionBase realIns = (UnaryMRInstructionBase) ins;
        dimOut.set(dims.get(realIns.input));
    } else if (ins instanceof MMTSJMRInstruction) {
        MMTSJMRInstruction mmtsj = (MMTSJMRInstruction) ins;
        MMTSJType tstype = mmtsj.getMMTSJType();
        MatrixCharacteristics mc = dims.get(mmtsj.input);
        dimOut.set(tstype.isLeft() ? mc.numColumns : mc.numRows, tstype.isLeft() ? mc.numColumns : mc.numRows, mc.numRowsPerBlock, mc.numColumnsPerBlock);
    } else if (ins instanceof PMMJMRInstruction) {
        PMMJMRInstruction pmmins = (PMMJMRInstruction) ins;
        MatrixCharacteristics mc = dims.get(pmmins.input2);
        dimOut.set(pmmins.getNumRows(), mc.numColumns, mc.numRowsPerBlock, mc.numColumnsPerBlock);
    } else if (ins instanceof RemoveEmptyMRInstruction) {
        RemoveEmptyMRInstruction realIns = (RemoveEmptyMRInstruction) ins;
        MatrixCharacteristics mc = dims.get(realIns.input1);
        long min = realIns.isEmptyReturn() ? 1 : 0;
        if (realIns.isRemoveRows())
            dimOut.set(Math.max(realIns.getOutputLen(), min), mc.getCols(), mc.numRowsPerBlock, mc.numColumnsPerBlock);
        else
            dimOut.set(mc.getRows(), Math.max(realIns.getOutputLen(), min), mc.numRowsPerBlock, mc.numColumnsPerBlock);
    } else if (// needs to be checked before binary
    ins instanceof UaggOuterChainInstruction) {
        UaggOuterChainInstruction realIns = (UaggOuterChainInstruction) ins;
        MatrixCharacteristics mc1 = dims.get(realIns.input1);
        MatrixCharacteristics mc2 = dims.get(realIns.input2);
        realIns.computeOutputCharacteristics(mc1, mc2, dimOut);
    } else if (ins instanceof GroupedAggregateMInstruction) {
        GroupedAggregateMInstruction realIns = (GroupedAggregateMInstruction) ins;
        MatrixCharacteristics mc1 = dims.get(realIns.input1);
        realIns.computeOutputCharacteristics(mc1, dimOut);
    } else if (ins instanceof BinaryInstruction || ins instanceof BinaryMInstruction || ins instanceof CombineBinaryInstruction) {
        BinaryMRInstructionBase realIns = (BinaryMRInstructionBase) ins;
        MatrixCharacteristics mc1 = dims.get(realIns.input1);
        MatrixCharacteristics mc2 = dims.get(realIns.input2);
        if (mc1.getRows() > 1 && mc1.getCols() == 1 && mc2.getRows() == 1 && // outer
        mc2.getCols() > 1) {
            dimOut.set(mc1.getRows(), mc2.getCols(), mc1.getRowsPerBlock(), mc2.getColsPerBlock());
        } else {
            // default case
            dimOut.set(mc1);
        }
    } else if (ins instanceof TernaryInstruction) {
        dimOut.set(dims.get(ins.getInputIndexes()[0]));
    } else if (ins instanceof CombineTernaryInstruction) {
        CtableInstruction realIns = (CtableInstruction) ins;
        dimOut.set(dims.get(realIns.input1));
    } else if (ins instanceof CombineUnaryInstruction) {
        dimOut.set(dims.get(((CombineUnaryInstruction) ins).input));
    } else if (ins instanceof CM_N_COVInstruction || ins instanceof GroupedAggregateInstruction) {
        dimOut.set(1, 1, 1, 1);
    } else if (ins instanceof RangeBasedReIndexInstruction) {
        RangeBasedReIndexInstruction realIns = (RangeBasedReIndexInstruction) ins;
        MatrixCharacteristics dimIn = dims.get(realIns.input);
        realIns.computeOutputCharacteristics(dimIn, dimOut);
    } else if (ins instanceof CtableInstruction) {
        CtableInstruction realIns = (CtableInstruction) ins;
        MatrixCharacteristics in_dim = dims.get(realIns.input1);
        dimOut.set(realIns.getOutputDim1(), realIns.getOutputDim2(), in_dim.numRowsPerBlock, in_dim.numColumnsPerBlock);
    } else {
        /*
			 * if ins is none of the above cases then we assume that dim_out dimensions are unknown
			 */
        dimOut.numRows = -1;
        dimOut.numColumns = -1;
        dimOut.numRowsPerBlock = 1;
        dimOut.numColumnsPerBlock = 1;
    }
}