Examples with GroupedAggregateInstruction org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction used on opensource projects

Example 6 with GroupedAggregateInstruction

use of org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction in project systemml by apache.

the class GroupedAggMRReducer method reduce.

@Override
public void reduce(TaggedMatrixIndexes key, Iterator<WeightedCell> values, OutputCollector<MatrixIndexes, MatrixCell> out, Reporter report) throws IOException {
    commonSetup(report);
    // get operator
    GroupedAggregateInstruction ins = grpaggInstructions.get(key.getTag());
    Operator op = ins.getOperator();
    try {
        if (// all, but sum
        op instanceof CMOperator) {
            cmObj.reset();
            CM lcmFn = cmFn.get(key.getTag());
            while (values.hasNext()) {
                WeightedCell value = values.next();
                lcmFn.execute(cmObj, value.getValue(), value.getWeight());
            }
            outCell.setValue(cmObj.getRequiredResult(op));
        } else if (// sum
        op instanceof AggregateOperator) {
            AggregateOperator aggop = (AggregateOperator) op;
            if (aggop.correctionExists) {
                KahanObject buffer = new KahanObject(aggop.initialValue, 0);
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    aggop.increOp.fn.execute(buffer, value.getValue() * value.getWeight());
                }
                outCell.setValue(buffer._sum);
            } else {
                double v = aggop.initialValue;
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    v = aggop.increOp.fn.execute(v, value.getValue() * value.getWeight());
                }
                outCell.setValue(v);
            }
        } else
            throw new IOException("Unsupported operator in instruction: " + ins);
    } catch (Exception ex) {
        throw new IOException(ex);
    }
    outIndex.setIndexes(key.getBaseObject());
    cachedValues.reset();
    cachedValues.set(key.getTag(), outIndex, outCell);
    processReducerInstructions();
    // output the final result matrices
    outputResultsFromCachedValues(report);
}

Example 7 with GroupedAggregateInstruction

use of org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction in project incubator-systemml by apache.

the class GroupedAggMRReducer method configure.

@Override
public void configure(JobConf job) {
    super.configure(job);
    try {
        GroupedAggregateInstruction[] grpaggIns = MRJobConfiguration.getGroupedAggregateInstructions(job);
        if (grpaggIns == null)
            throw new RuntimeException("no GroupAggregate Instructions found!");
        for (GroupedAggregateInstruction ins : grpaggIns) {
            grpaggInstructions.put(ins.output, ins);
            if (ins.getOperator() instanceof CMOperator)
                cmFn.put(ins.output, CM.getCMFnObject(((CMOperator) ins.getOperator()).getAggOpType()));
            outputIndexesMapping.put(ins.output, getOutputIndexes(ins.output));
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 8 with GroupedAggregateInstruction

use of org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction in project systemml by apache.

the class GroupedAggMRReducer method configure.

@Override
public void configure(JobConf job) {
    super.configure(job);
    try {
        GroupedAggregateInstruction[] grpaggIns = MRJobConfiguration.getGroupedAggregateInstructions(job);
        if (grpaggIns == null)
            throw new RuntimeException("no GroupAggregate Instructions found!");
        for (GroupedAggregateInstruction ins : grpaggIns) {
            grpaggInstructions.put(ins.output, ins);
            if (ins.getOperator() instanceof CMOperator)
                cmFn.put(ins.output, CM.getCMFnObject(((CMOperator) ins.getOperator()).getAggOpType()));
            outputIndexesMapping.put(ins.output, getOutputIndexes(ins.output));
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 9 with GroupedAggregateInstruction

use of org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction in project systemml by apache.

the class GroupedAggMRCombiner method reduce.

@Override
public void reduce(TaggedMatrixIndexes key, Iterator<WeightedCell> values, OutputCollector<TaggedMatrixIndexes, WeightedCell> out, Reporter reporter) throws IOException {
    long start = System.currentTimeMillis();
    // get aggregate operator
    GroupedAggregateInstruction ins = grpaggInstructions.get(key.getTag());
    Operator op = ins.getOperator();
    boolean isPartialAgg = true;
    // combine iterator to single value
    try {
        if (// everything except sum
        op instanceof CMOperator) {
            if (((CMOperator) op).isPartialAggregateOperator()) {
                cmObj.reset();
                CM lcmFn = cmFn.get(key.getTag());
                // partial aggregate cm operator
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    lcmFn.execute(cmObj, value.getValue(), value.getWeight());
                }
                outCell.setValue(cmObj.getRequiredPartialResult(op));
                outCell.setWeight(cmObj.getWeight());
            } else // forward tuples to reducer
            {
                isPartialAgg = false;
                while (values.hasNext()) out.collect(key, values.next());
            }
        } else if (// sum
        op instanceof AggregateOperator) {
            AggregateOperator aggop = (AggregateOperator) op;
            if (aggop.correctionExists) {
                KahanObject buffer = new KahanObject(aggop.initialValue, 0);
                KahanPlus.getKahanPlusFnObject();
                // partial aggregate with correction
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    aggop.increOp.fn.execute(buffer, value.getValue() * value.getWeight());
                }
                outCell.setValue(buffer._sum);
                outCell.setWeight(1);
            } else // no correction
            {
                double v = aggop.initialValue;
                // partial aggregate without correction
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    v = aggop.increOp.fn.execute(v, value.getValue() * value.getWeight());
                }
                outCell.setValue(v);
                outCell.setWeight(1);
            }
        } else
            throw new IOException("Unsupported operator in instruction: " + ins);
    } catch (Exception ex) {
        throw new IOException(ex);
    }
    // collect the output (to reducer)
    if (isPartialAgg)
        out.collect(key, outCell);
    reporter.incrCounter(Counters.COMBINE_OR_REDUCE_TIME, System.currentTimeMillis() - start);
}

Example 10 with GroupedAggregateInstruction

use of org.apache.sysml.runtime.instructions.mr.GroupedAggregateInstruction in project systemml by apache.

the class GroupedAggMRMapper method configure.

@Override
public void configure(JobConf job) {
    super.configure(job);
    try {
        GroupedAggregateInstruction[] grpaggIns = MRJobConfiguration.getGroupedAggregateInstructions(job);
        if (grpaggIns == null)
            throw new RuntimeException("no GroupAggregate Instructions found!");
        ArrayList<GroupedAggregateInstruction> vec = new ArrayList<>();
        for (int i = 0; i < representativeMatrixes.size(); i++) {
            byte input = representativeMatrixes.get(i);
            for (GroupedAggregateInstruction ins : grpaggIns) if (ins.input == input)
                vec.add(ins);
            groupAgg_instructions.add(vec);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}