Examples with AggregateUnaryOperator org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator used on opensource projects

Example 21 with AggregateUnaryOperator

use of org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator in project incubator-systemml by apache.

the class AggregateUnarySPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    MatrixCharacteristics mc = sec.getMatrixCharacteristics(input1.getName());
    // get input
    JavaPairRDD<MatrixIndexes, MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(input1.getName());
    JavaPairRDD<MatrixIndexes, MatrixBlock> out = in;
    // filter input blocks for trace
    if (getOpcode().equalsIgnoreCase("uaktrace"))
        out = out.filter(new FilterDiagBlocksFunction());
    // execute unary aggregate operation
    AggregateUnaryOperator auop = (AggregateUnaryOperator) _optr;
    AggregateOperator aggop = _aop;
    // perform aggregation if necessary and put output into symbol table
    if (_aggtype == SparkAggType.SINGLE_BLOCK) {
        JavaRDD<MatrixBlock> out2 = out.map(new RDDUAggFunction2(auop, mc.getRowsPerBlock(), mc.getColsPerBlock()));
        MatrixBlock out3 = RDDAggregateUtils.aggStable(out2, aggop);
        // drop correction after aggregation
        out3.dropLastRowsOrColumns(aggop.correctionLocation);
        // put output block into symbol table (no lineage because single block)
        // this also includes implicit maintenance of matrix characteristics
        sec.setMatrixOutput(output.getName(), out3, getExtendedOpcode());
    } else // MULTI_BLOCK or NONE
    {
        if (_aggtype == SparkAggType.NONE) {
            // in case of no block aggregation, we always drop the correction as well as
            // use a partitioning-preserving mapvalues
            out = out.mapValues(new RDDUAggValueFunction(auop, mc.getRowsPerBlock(), mc.getColsPerBlock()));
        } else if (_aggtype == SparkAggType.MULTI_BLOCK) {
            // in case of multi-block aggregation, we always keep the correction
            out = out.mapToPair(new RDDUAggFunction(auop, mc.getRowsPerBlock(), mc.getColsPerBlock()));
            out = RDDAggregateUtils.aggByKeyStable(out, aggop, false);
            // partitioning, drop correction via partitioning-preserving mapvalues)
            if (auop.aggOp.correctionExists)
                out = out.mapValues(new AggregateDropCorrectionFunction(aggop));
        }
        // put output RDD handle into symbol table
        updateUnaryAggOutputMatrixCharacteristics(sec, auop.indexFn);
        sec.setRDDHandleForVariable(output.getName(), out);
        sec.addLineageRDD(output.getName(), input1.getName());
    }
}

Example 22 with AggregateUnaryOperator

use of org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator in project incubator-systemml by apache.

the class CumulativeAggregateSPInstruction method parseInstruction.

public static CumulativeAggregateSPInstruction parseInstruction(String str) {
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    InstructionUtils.checkNumFields(parts, 2);
    String opcode = parts[0];
    CPOperand in1 = new CPOperand(parts[1]);
    CPOperand out = new CPOperand(parts[2]);
    AggregateUnaryOperator aggun = InstructionUtils.parseCumulativeAggregateUnaryOperator(opcode);
    return new CumulativeAggregateSPInstruction(aggun, in1, out, opcode, str);
}

Example 23 with AggregateUnaryOperator

use of org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator in project incubator-systemml by apache.

the class LibMatrixAgg method cumaggregateUnaryMatrix.

public static MatrixBlock cumaggregateUnaryMatrix(MatrixBlock in, MatrixBlock out, UnaryOperator uop, int k) {
    AggregateUnaryOperator uaop = InstructionUtils.parseBasicCumulativeAggregateUnaryOperator(uop);
    // fall back to sequential if necessary or agg not supported
    if (k <= 1 || (long) in.rlen * in.clen < PAR_NUMCELL_THRESHOLD || in.rlen <= k || out.clen * 8 * k > PAR_INTERMEDIATE_SIZE_THRESHOLD || uaop == null || !out.isThreadSafe()) {
        return cumaggregateUnaryMatrix(in, out, uop);
    }
    // prepare meta data
    AggType aggtype = getAggType(uop);
    final int m = in.rlen;
    final int m2 = out.rlen;
    final int n2 = out.clen;
    final int mk = aggtype == AggType.CUM_KAHAN_SUM ? 2 : 1;
    // filter empty input blocks (incl special handling for sparse-unsafe operations)
    if (in.isEmptyBlock(false)) {
        return aggregateUnaryMatrixEmpty(in, out, aggtype, null);
    }
    // Timing time = new Timing(true);
    // allocate output arrays (if required)
    // always dense
    out.reset(m2, n2, false);
    out.allocateDenseBlock();
    // (currently: always parallelization over number of rows)
    try {
        ExecutorService pool = CommonThreadPool.get(k);
        int blklen = (int) (Math.ceil((double) m / k));
        // step 1: compute aggregates per row partition
        AggType uaoptype = getAggType(uaop);
        ArrayList<PartialAggTask> tasks = new ArrayList<>();
        for (int i = 0; i < k & i * blklen < m; i++) tasks.add(new PartialAggTask(in, new MatrixBlock(mk, n2, false), uaoptype, uaop, i * blklen, Math.min((i + 1) * blklen, m)));
        List<Future<Object>> taskret = pool.invokeAll(tasks);
        for (Future<Object> task : taskret) // check for errors
        task.get();
        // step 2: cumulative aggregate of partition aggregates
        MatrixBlock tmp = new MatrixBlock(tasks.size(), n2, false);
        for (int i = 0; i < tasks.size(); i++) {
            MatrixBlock row = tasks.get(i).getResult();
            if (uaop.aggOp.correctionExists)
                row.dropLastRowsOrColumns(uaop.aggOp.correctionLocation);
            tmp.leftIndexingOperations(row, i, i, 0, n2 - 1, tmp, UpdateType.INPLACE_PINNED);
        }
        MatrixBlock tmp2 = cumaggregateUnaryMatrix(tmp, new MatrixBlock(tasks.size(), n2, false), uop);
        // step 3: compute final cumulative aggregate
        ArrayList<CumAggTask> tasks2 = new ArrayList<>();
        for (int i = 0; i < k & i * blklen < m; i++) {
            double[] agg = (i == 0) ? null : DataConverter.convertToDoubleVector(tmp2.slice(i - 1, i - 1, 0, n2 - 1, new MatrixBlock()), false);
            tasks2.add(new CumAggTask(in, agg, out, aggtype, uop, i * blklen, Math.min((i + 1) * blklen, m)));
        }
        List<Future<Long>> taskret2 = pool.invokeAll(tasks2);
        pool.shutdown();
        // step 4: aggregate nnz
        out.nonZeros = 0;
        for (Future<Long> task : taskret2) out.nonZeros += task.get();
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    // cleanup output and change representation (if necessary)
    out.examSparsity();
    return out;
}

Example 24 with AggregateUnaryOperator

use of org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator in project incubator-systemml by apache.

the class AggregateUnaryCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    String output_name = output.getName();
    String opcode = getOpcode();
    if (// nrow/ncol/length
    _type.isMeta()) {
        // check existence of input variable
        if (!ec.getVariables().keySet().contains(input1.getName()))
            throw new DMLRuntimeException("Variable '" + input1.getName() + "' does not exist.");
        // get meta data information
        MatrixCharacteristics mc = ec.getMatrixCharacteristics(input1.getName());
        long rval = getSizeMetaData(_type, mc);
        // Note: check on matrix characteristics to cover incorrect length (-1*-1 -> 1)
        if (// invalid nrow/ncol/length
        !mc.dimsKnown()) {
            if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE || (input1.getDataType() == DataType.FRAME && OptimizerUtils.isHadoopExecutionMode())) {
                if (OptimizerUtils.isHadoopExecutionMode())
                    LOG.warn("Reading csv input frame of unkown size into memory for '" + opcode + "'.");
                // read the input matrix/frame and explicitly refresh meta data
                CacheableData<?> obj = ec.getCacheableData(input1.getName());
                obj.acquireRead();
                obj.refreshMetaData();
                obj.release();
                // update meta data information
                mc = ec.getMatrixCharacteristics(input1.getName());
                rval = getSizeMetaData(_type, mc);
            } else {
                throw new DMLRuntimeException("Invalid meta data returned by '" + opcode + "': " + rval + ":" + instString);
            }
        }
        // create and set output scalar
        ec.setScalarOutput(output_name, new IntObject(rval));
    } else {
        // DEFAULT
        MatrixBlock matBlock = ec.getMatrixInput(input1.getName());
        AggregateUnaryOperator au_op = (AggregateUnaryOperator) _optr;
        MatrixBlock resultBlock = (MatrixBlock) matBlock.aggregateUnaryOperations(au_op, new MatrixBlock(), matBlock.getNumRows(), matBlock.getNumColumns(), new MatrixIndexes(1, 1), true);
        ec.releaseMatrixInput(input1.getName());
        if (output.getDataType() == DataType.SCALAR) {
            DoubleObject ret = new DoubleObject(resultBlock.getValue(0, 0));
            ec.setScalarOutput(output_name, ret);
        } else {
            // since the computed value is a scalar, allocate a "temp" output matrix
            ec.setMatrixOutput(output_name, resultBlock);
        }
    }
}

Example 25 with AggregateUnaryOperator

use of org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator in project incubator-systemml by apache.

the class MatrixBlock method max.

/**
 * Wrapper method for reduceall-max of a matrix.
 *
 * @return ?
 */
public double max() {
    // construct operator
    AggregateOperator aop = new AggregateOperator(Double.NEGATIVE_INFINITY, Builtin.getBuiltinFnObject("max"));
    AggregateUnaryOperator auop = new AggregateUnaryOperator(aop, ReduceAll.getReduceAllFnObject());
    // execute operation
    MatrixBlock out = new MatrixBlock(1, 1, false);
    LibMatrixAgg.aggregateUnaryMatrix(this, out, auop);
    return out.quickGetValue(0, 0);
}