Examples with DoubleObject org.apache.sysml.runtime.instructions.cp.DoubleObject used on opensource projects

Example 1 with DoubleObject

use of org.apache.sysml.runtime.instructions.cp.DoubleObject in project incubator-systemml by apache.

the class QuaternarySPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    QuaternaryOperator qop = (QuaternaryOperator) _optr;
    //tracking of rdds and broadcasts (for lineage maintenance)
    ArrayList<String> rddVars = new ArrayList<String>();
    ArrayList<String> bcVars = new ArrayList<String>();
    JavaPairRDD<MatrixIndexes, MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(input1.getName());
    JavaPairRDD<MatrixIndexes, MatrixBlock> out = null;
    MatrixCharacteristics inMc = sec.getMatrixCharacteristics(input1.getName());
    long rlen = inMc.getRows();
    long clen = inMc.getCols();
    int brlen = inMc.getRowsPerBlock();
    int bclen = inMc.getColsPerBlock();
    //(map/redwsloss, map/redwcemm); safe because theses ops produce a scalar
    if (qop.wtype1 != null || qop.wtype4 != null) {
        in = in.filter(new FilterNonEmptyBlocksFunction());
    }
    //map-side only operation (one rdd input, two broadcasts)
    if (WeightedSquaredLoss.OPCODE.equalsIgnoreCase(getOpcode()) || WeightedSigmoid.OPCODE.equalsIgnoreCase(getOpcode()) || WeightedDivMM.OPCODE.equalsIgnoreCase(getOpcode()) || WeightedCrossEntropy.OPCODE.equalsIgnoreCase(getOpcode()) || WeightedUnaryMM.OPCODE.equalsIgnoreCase(getOpcode())) {
        PartitionedBroadcast<MatrixBlock> bc1 = sec.getBroadcastForVariable(input2.getName());
        PartitionedBroadcast<MatrixBlock> bc2 = sec.getBroadcastForVariable(input3.getName());
        //partitioning-preserving mappartitions (key access required for broadcast loopkup)
        //only wdivmm changes keys
        boolean noKeyChange = (qop.wtype3 == null || qop.wtype3.isBasic());
        out = in.mapPartitionsToPair(new RDDQuaternaryFunction1(qop, bc1, bc2), noKeyChange);
        rddVars.add(input1.getName());
        bcVars.add(input2.getName());
        bcVars.add(input3.getName());
    } else //reduce-side operation (two/three/four rdd inputs, zero/one/two broadcasts)
    {
        PartitionedBroadcast<MatrixBlock> bc1 = _cacheU ? sec.getBroadcastForVariable(input2.getName()) : null;
        PartitionedBroadcast<MatrixBlock> bc2 = _cacheV ? sec.getBroadcastForVariable(input3.getName()) : null;
        JavaPairRDD<MatrixIndexes, MatrixBlock> inU = (!_cacheU) ? sec.getBinaryBlockRDDHandleForVariable(input2.getName()) : null;
        JavaPairRDD<MatrixIndexes, MatrixBlock> inV = (!_cacheV) ? sec.getBinaryBlockRDDHandleForVariable(input3.getName()) : null;
        JavaPairRDD<MatrixIndexes, MatrixBlock> inW = (qop.hasFourInputs() && !_input4.isLiteral()) ? sec.getBinaryBlockRDDHandleForVariable(_input4.getName()) : null;
        //preparation of transposed and replicated U
        if (inU != null)
            inU = inU.flatMapToPair(new ReplicateBlocksFunction(clen, bclen, true));
        //preparation of transposed and replicated V
        if (inV != null)
            inV = inV.mapToPair(new TransposeFactorIndexesFunction()).flatMapToPair(new ReplicateBlocksFunction(rlen, brlen, false));
        //functions calls w/ two rdd inputs		
        if (inU != null && inV == null && inW == null)
            out = in.join(inU).mapToPair(new RDDQuaternaryFunction2(qop, bc1, bc2));
        else if (inU == null && inV != null && inW == null)
            out = in.join(inV).mapToPair(new RDDQuaternaryFunction2(qop, bc1, bc2));
        else if (inU == null && inV == null && inW != null)
            out = in.join(inW).mapToPair(new RDDQuaternaryFunction2(qop, bc1, bc2));
        else //function calls w/ three rdd inputs
        if (inU != null && inV != null && inW == null)
            out = in.join(inU).join(inV).mapToPair(new RDDQuaternaryFunction3(qop, bc1, bc2));
        else if (inU != null && inV == null && inW != null)
            out = in.join(inU).join(inW).mapToPair(new RDDQuaternaryFunction3(qop, bc1, bc2));
        else if (inU == null && inV != null && inW != null)
            out = in.join(inV).join(inW).mapToPair(new RDDQuaternaryFunction3(qop, bc1, bc2));
        else if (inU == null && inV == null && inW == null) {
            out = in.mapPartitionsToPair(new RDDQuaternaryFunction1(qop, bc1, bc2), false);
        } else
            //function call w/ four rdd inputs
            //need keys in case of wdivmm 
            out = in.join(inU).join(inV).join(inW).mapToPair(new RDDQuaternaryFunction4(qop));
        //keep variable names for lineage maintenance
        if (inU == null)
            bcVars.add(input2.getName());
        else
            rddVars.add(input2.getName());
        if (inV == null)
            bcVars.add(input3.getName());
        else
            rddVars.add(input3.getName());
        if (inW != null)
            rddVars.add(_input4.getName());
    }
    //output handling, incl aggregation
    if (//map/redwsloss, map/redwcemm
    qop.wtype1 != null || qop.wtype4 != null) {
        //full aggregate and cast to scalar
        MatrixBlock tmp = RDDAggregateUtils.sumStable(out);
        DoubleObject ret = new DoubleObject(tmp.getValue(0, 0));
        sec.setVariable(output.getName(), ret);
    } else //map/redwsigmoid, map/redwdivmm, map/redwumm 
    {
        //aggregation if required (map/redwdivmm)
        if (qop.wtype3 != null && !qop.wtype3.isBasic())
            out = RDDAggregateUtils.sumByKeyStable(out, false);
        //put output RDD handle into symbol table
        sec.setRDDHandleForVariable(output.getName(), out);
        //maintain lineage information for output rdd
        for (String rddVar : rddVars) sec.addLineageRDD(output.getName(), rddVar);
        for (String bcVar : bcVars) sec.addLineageBroadcast(output.getName(), bcVar);
        //update matrix characteristics
        updateOutputMatrixCharacteristics(sec, qop);
    }
}

Example 2 with DoubleObject

use of org.apache.sysml.runtime.instructions.cp.DoubleObject in project incubator-systemml by apache.

the class SpoofOuterProduct method execute.

@Override
public ScalarObject execute(ArrayList<MatrixBlock> inputs, ArrayList<ScalarObject> scalarObjects, int numThreads) throws DMLRuntimeException {
    //sanity check
    if (inputs == null || inputs.size() < 3)
        throw new RuntimeException("Invalid input arguments.");
    if (inputs.get(0).isEmptyBlock(false))
        return new DoubleObject(0);
    //input preparation
    double[][] ab = prepInputMatricesDense(inputs, 1, 2);
    double[][] b = prepInputMatricesDense(inputs, 3);
    double[] scalars = prepInputScalars(scalarObjects);
    //core sequential execute
    final int m = inputs.get(0).getNumRows();
    final int n = inputs.get(0).getNumColumns();
    // rank
    final int k = inputs.get(1).getNumColumns();
    double sum = 0;
    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<ParOuterProdAggTask> tasks = new ArrayList<ParOuterProdAggTask>();
        //create tasks (for wdivmm-left, parallelization over columns;
        //for wdivmm-right, parallelization over rows; both ensure disjoint results)
        int blklen = (int) (Math.ceil((double) m / numThreads));
        for (int i = 0; i < numThreads & i * blklen < m; i++) tasks.add(new ParOuterProdAggTask(inputs.get(0), ab[0], ab[1], b, scalars, m, n, k, _outerProductType, i * blklen, Math.min((i + 1) * blklen, m), 0, n));
        //execute tasks
        List<Future<Double>> taskret = pool.invokeAll(tasks);
        pool.shutdown();
        for (Future<Double> task : taskret) sum += task.get();
    } catch (Exception e) {
        throw new DMLRuntimeException(e);
    }
    return new DoubleObject(sum);
}

Example 3 with DoubleObject

use of org.apache.sysml.runtime.instructions.cp.DoubleObject in project incubator-systemml by apache.

the class SpoofOuterProduct method execute.

@Override
public ScalarObject execute(ArrayList<MatrixBlock> inputs, ArrayList<ScalarObject> scalarObjects) throws DMLRuntimeException {
    //sanity check
    if (inputs == null || inputs.size() < 3)
        throw new RuntimeException("Invalid input arguments.");
    if (inputs.get(0).isEmptyBlock(false))
        return new DoubleObject(0);
    //input preparation
    double[][] ab = prepInputMatricesDense(inputs, 1, 2);
    double[][] b = prepInputMatricesDense(inputs, 3);
    double[] scalars = prepInputScalars(scalarObjects);
    //core sequential execute
    final int m = inputs.get(0).getNumRows();
    final int n = inputs.get(0).getNumColumns();
    // rank
    final int k = inputs.get(1).getNumColumns();
    MatrixBlock a = inputs.get(0);
    MatrixBlock out = new MatrixBlock(1, 1, false);
    out.allocateDenseBlock();
    if (a instanceof CompressedMatrixBlock)
        executeCellwiseCompressed((CompressedMatrixBlock) a, ab[0], ab[1], b, scalars, out, m, n, k, _outerProductType, 0, m, 0, n);
    else if (!a.isInSparseFormat())
        executeCellwiseDense(a.getDenseBlock(), ab[0], ab[1], b, scalars, out.getDenseBlock(), m, n, k, _outerProductType, 0, m, 0, n);
    else
        executeCellwiseSparse(a.getSparseBlock(), ab[0], ab[1], b, scalars, out, m, n, k, a.getNonZeros(), _outerProductType, 0, m, 0, n);
    return new DoubleObject(out.getDenseBlock()[0]);
}

Example 4 with DoubleObject

use of org.apache.sysml.runtime.instructions.cp.DoubleObject in project incubator-systemml by apache.

the class AggregateTernarySPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    //get inputs
    MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName());
    JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(input1.getName());
    JavaPairRDD<MatrixIndexes, MatrixBlock> in2 = sec.getBinaryBlockRDDHandleForVariable(input2.getName());
    JavaPairRDD<MatrixIndexes, MatrixBlock> in3 = //matrix or literal 1
    input3.isLiteral() ? //matrix or literal 1
    null : sec.getBinaryBlockRDDHandleForVariable(input3.getName());
    //execute aggregate ternary operation
    AggregateTernaryOperator aggop = (AggregateTernaryOperator) _optr;
    JavaPairRDD<MatrixIndexes, MatrixBlock> out = null;
    if (in3 != null) {
        //3 inputs
        out = in1.join(in2).join(in3).mapToPair(new RDDAggregateTernaryFunction(aggop));
    } else {
        //2 inputs (third is literal 1)
        out = in1.join(in2).mapToPair(new RDDAggregateTernaryFunction2(aggop));
    }
    //aggregate partial results
    if (//tak+*
    aggop.indexFn instanceof ReduceAll) {
        //aggregate and create output (no lineage because scalar)	   
        MatrixBlock tmp = RDDAggregateUtils.sumStable(out.values());
        DoubleObject ret = new DoubleObject(tmp.getValue(0, 0));
        sec.setVariable(output.getName(), ret);
    } else if (//tack+* single block
    mcIn.dimsKnown() && mcIn.getCols() <= mcIn.getColsPerBlock()) {
        //single block aggregation and drop correction
        MatrixBlock ret = RDDAggregateUtils.aggStable(out, aggop.aggOp);
        ret.dropLastRowsOrColums(aggop.aggOp.correctionLocation);
        //put output block into symbol table (no lineage because single block)
        //this also includes implicit maintenance of matrix characteristics
        sec.setMatrixOutput(output.getName(), ret);
    } else //tack+* multi block
    {
        //multi-block aggregation and drop correction
        out = RDDAggregateUtils.aggByKeyStable(out, aggop.aggOp, false);
        out = out.mapValues(new AggregateDropCorrectionFunction(aggop.aggOp));
        //put output RDD handle into symbol table
        updateUnaryAggOutputMatrixCharacteristics(sec, aggop.indexFn);
        sec.setRDDHandleForVariable(output.getName(), out);
        sec.addLineageRDD(output.getName(), input1.getName());
        sec.addLineageRDD(output.getName(), input2.getName());
        if (in3 != null)
            sec.addLineageRDD(output.getName(), input3.getName());
    }
}

Example 5 with DoubleObject

use of org.apache.sysml.runtime.instructions.cp.DoubleObject in project incubator-systemml by apache.

the class CentralMomentSPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    //parse 'order' input argument 
    CPOperand scalarInput = (input3 == null ? input2 : input3);
    ScalarObject order = ec.getScalarInput(scalarInput.getName(), scalarInput.getValueType(), scalarInput.isLiteral());
    CMOperator cop = ((CMOperator) _optr);
    if (cop.getAggOpType() == AggregateOperationTypes.INVALID) {
        cop.setCMAggOp((int) order.getLongValue());
    }
    //get input
    JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(input1.getName());
    //process central moment instruction
    CM_COV_Object cmobj = null;
    if (//w/o weights
    input3 == null) {
        cmobj = in1.values().map(new RDDCMFunction(cop)).fold(new CM_COV_Object(), new RDDCMReduceFunction(cop));
    } else //with weights
    {
        JavaPairRDD<MatrixIndexes, MatrixBlock> in2 = sec.getBinaryBlockRDDHandleForVariable(input2.getName());
        cmobj = in1.join(in2).values().map(new RDDCMWeightsFunction(cop)).fold(new CM_COV_Object(), new RDDCMReduceFunction(cop));
    }
    //create scalar output (no lineage information required)
    double val = cmobj.getRequiredResult(_optr);
    DoubleObject ret = new DoubleObject(output.getName(), val);
    ec.setScalarOutput(output.getName(), ret);
}