Examples with Lop org.apache.sysml.lops.Lop used on opensource projects

Example 71 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Hop method constructAndSetCheckpointLopIfRequired.

private void constructAndSetCheckpointLopIfRequired() {
    // determine execution type
    ExecType et = ExecType.CP;
    if (OptimizerUtils.isSparkExecutionMode() && getDataType() != DataType.SCALAR) {
        // (2) avoid unnecessary creation of spark context (incl executors)
        if ((OptimizerUtils.isHybridExecutionMode() && hasValidCPDimsAndSize() && !OptimizerUtils.exceedsCachingThreshold(getDim2(), _outputMemEstimate)) || _etypeForced == ExecType.CP) {
            et = ExecType.CP;
        } else // default case
        {
            et = ExecType.SPARK;
        }
    }
    // add checkpoint lop to output if required
    if (_requiresCheckpoint && et != ExecType.CP) {
        try {
            // investigate need for serialized storage of large sparse matrices
            // (compile- instead of runtime-level for better debugging)
            boolean serializedStorage = false;
            if (getDataType() == DataType.MATRIX && dimsKnown(true)) {
                double matrixPSize = OptimizerUtils.estimatePartitionedSizeExactSparsity(_dim1, _dim2, _rows_in_block, _cols_in_block, _nnz);
                double dataCache = SparkExecutionContext.getDataMemoryBudget(true, true);
                serializedStorage = MatrixBlock.evalSparseFormatInMemory(_dim1, _dim2, _nnz) && // sparse in-memory does not fit in agg mem
                matrixPSize > dataCache && (OptimizerUtils.getSparsity(_dim1, _dim2, _nnz) < MatrixBlock.ULTRA_SPARSITY_TURN_POINT || // ultra-sparse or sparse w/o csr
                !Checkpoint.CHECKPOINT_SPARSE_CSR);
            } else if (!dimsKnown(true)) {
                setRequiresRecompile();
            }
            // construct checkpoint w/ right storage level
            Lop input = getLops();
            Lop chkpoint = new Checkpoint(input, getDataType(), getValueType(), serializedStorage ? Checkpoint.getSerializeStorageLevelString() : Checkpoint.getDefaultStorageLevelString());
            setOutputDimensions(chkpoint);
            setLineNumbers(chkpoint);
            setLops(chkpoint);
        } catch (LopsException ex) {
            throw new HopsException(ex);
        }
    }
}

Example 72 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Hop method constructAndSetReblockLopIfRequired.

private void constructAndSetReblockLopIfRequired() {
    // determine execution type
    ExecType et = ExecType.CP;
    if (DMLScript.rtplatform != RUNTIME_PLATFORM.SINGLE_NODE && !(getDataType() == DataType.SCALAR)) {
        et = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    }
    // add reblock lop to output if required
    if (_requiresReblock && et != ExecType.CP) {
        Lop input = getLops();
        Lop reblock = null;
        try {
            if (// CSV
            this instanceof DataOp && ((DataOp) this).getDataOpType() == DataOpTypes.PERSISTENTREAD && ((DataOp) this).getInputFormatType() == FileFormatTypes.CSV) {
                reblock = new CSVReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), et);
            } else // TEXT / MM / BINARYBLOCK / BINARYCELL
            {
                reblock = new ReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), _outputEmptyBlocks, et);
            }
        } catch (LopsException ex) {
            throw new HopsException(ex);
        }
        setOutputDimensions(reblock);
        setLineNumbers(reblock);
        setLops(reblock);
    }
}

Example 73 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Hop method createOffsetLop.

public static Lop createOffsetLop(Hop hop, boolean repCols) {
    Lop offset = null;
    if (ConfigurationManager.isDynamicRecompilation() && hop.dimsKnown()) {
        // If dynamic recompilation is enabled and dims are known, we can replace the ncol with
        // a literal in order to increase the piggybacking potential. This is safe because append
        // is always marked for recompilation and hence, we have propagated the exact dimensions.
        offset = Data.createLiteralLop(ValueType.INT, String.valueOf(repCols ? hop.getDim2() : hop.getDim1()));
    } else {
        offset = new UnaryCP(hop.constructLops(), repCols ? UnaryCP.OperationTypes.NCOL : UnaryCP.OperationTypes.NROW, DataType.SCALAR, ValueType.INT);
    }
    offset.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
    offset.setAllPositions(hop.getFilename(), hop.getBeginLine(), hop.getBeginColumn(), hop.getEndLine(), hop.getEndColumn());
    return offset;
}

Example 74 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class LeftIndexingOp method constructLops.

@Override
public Lop constructLops() {
    // return already created lops
    if (getLops() != null)
        return getLops();
    try {
        ExecType et = optFindExecType();
        if (et == ExecType.MR) {
            // the right matrix is reindexed
            Lop top = getInput().get(2).constructLops();
            Lop bottom = getInput().get(3).constructLops();
            Lop left = getInput().get(4).constructLops();
            Lop right = getInput().get(5).constructLops();
            // right hand matrix
            Lop nrow = new UnaryCP(getInput().get(0).constructLops(), OperationTypes.NROW, DataType.SCALAR, ValueType.INT);
            Lop ncol = new UnaryCP(getInput().get(0).constructLops(), OperationTypes.NCOL, DataType.SCALAR, ValueType.INT);
            Lop rightInput = null;
            if (isRightHandSideScalar()) {
                // insert cast to matrix if necessary (for reuse MR runtime)
                rightInput = new UnaryCP(getInput().get(1).constructLops(), OperationTypes.CAST_AS_MATRIX, DataType.MATRIX, ValueType.DOUBLE);
                rightInput.getOutputParameters().setDimensions(1L, 1L, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), -1L);
            } else
                rightInput = getInput().get(1).constructLops();
            RightIndex reindex = new RightIndex(rightInput, top, bottom, left, right, nrow, ncol, getDataType(), getValueType(), et, true);
            reindex.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(reindex);
            Group group1 = new Group(reindex, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
            group1.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(group1);
            // the left matrix is zeroed out
            ZeroOut zeroout = new ZeroOut(getInput().get(0).constructLops(), top, bottom, left, right, getInput().get(0).getDim1(), getInput().get(0).getDim2(), getDataType(), getValueType(), et);
            zeroout.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(zeroout);
            Group group2 = new Group(zeroout, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
            group2.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(group2);
            Binary binary = new Binary(group1, group2, HopsOpOp2LopsB.get(Hop.OpOp2.PLUS), getDataType(), getValueType(), et);
            binary.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(binary);
            setLops(binary);
        } else if (et == ExecType.SPARK) {
            Hop left = getInput().get(0);
            Hop right = getInput().get(1);
            LeftIndexingMethod method = getOptMethodLeftIndexingMethod(left.getDim1(), left.getDim2(), left.getRowsInBlock(), left.getColsInBlock(), left.getNnz(), right.getDim1(), right.getDim2(), right.getNnz(), right.getDataType());
            // insert cast to matrix if necessary (for reuse broadcast runtime)
            Lop rightInput = right.constructLops();
            if (isRightHandSideScalar()) {
                rightInput = new UnaryCP(rightInput, (left.getDataType() == DataType.MATRIX ? OperationTypes.CAST_AS_MATRIX : OperationTypes.CAST_AS_FRAME), left.getDataType(), right.getValueType());
                long bsize = ConfigurationManager.getBlocksize();
                rightInput.getOutputParameters().setDimensions(1, 1, bsize, bsize, -1);
            }
            LeftIndex leftIndexLop = new LeftIndex(left.constructLops(), rightInput, getInput().get(2).constructLops(), getInput().get(3).constructLops(), getInput().get(4).constructLops(), getInput().get(5).constructLops(), getDataType(), getValueType(), et, getSpLixCacheType(method));
            setOutputDimensions(leftIndexLop);
            setLineNumbers(leftIndexLop);
            setLops(leftIndexLop);
        } else {
            LeftIndex left = new LeftIndex(getInput().get(0).constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(), getInput().get(3).constructLops(), getInput().get(4).constructLops(), getInput().get(5).constructLops(), getDataType(), getValueType(), et);
            setOutputDimensions(left);
            setLineNumbers(left);
            setLops(left);
        }
    } catch (Exception e) {
        throw new HopsException(this.printErrorLocation() + "In LeftIndexingOp Hop, error in constructing Lops ", e);
    }
    // add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 75 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class QuaternaryOp method constructMRLopsWeightedSquaredLoss.

private void constructMRLopsWeightedSquaredLoss(WeightsType wtype) {
    // NOTE: the common case for wsloss are factors U/V with a rank of 10s to 100s; the current runtime only
    // supports single block outer products (U/V rank <= blocksize, i.e., 1000 by default); we enforce this
    // by applying the hop rewrite for Weighted Squared Loss only if this constraint holds.
    Hop X = getInput().get(0);
    Hop U = getInput().get(1);
    Hop V = getInput().get(2);
    Hop W = getInput().get(3);
    // MR operator selection, part1
    // size U
    double m1Size = OptimizerUtils.estimateSize(U.getDim1(), U.getDim2());
    // size V
    double m2Size = OptimizerUtils.estimateSize(V.getDim1(), V.getDim2());
    boolean isMapWsloss = (!wtype.hasFourInputs() && m1Size + m2Size < OptimizerUtils.getRemoteMemBudgetMap(true));
    if (// broadcast
    !FORCE_REPLICATION && isMapWsloss) {
        // partitioning of U
        boolean needPartU = !U.dimsKnown() || U.getDim1() * U.getDim2() > DistributedCacheInput.PARTITION_SIZE;
        Lop lU = U.constructLops();
        if (needPartU) {
            // requires partitioning
            lU = new DataPartition(lU, DataType.MATRIX, ValueType.DOUBLE, (m1Size > OptimizerUtils.getLocalMemBudget()) ? ExecType.MR : ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
            lU.getOutputParameters().setDimensions(U.getDim1(), U.getDim2(), getRowsInBlock(), getColsInBlock(), U.getNnz());
            setLineNumbers(lU);
        }
        // partitioning of V
        boolean needPartV = !V.dimsKnown() || V.getDim1() * V.getDim2() > DistributedCacheInput.PARTITION_SIZE;
        Lop lV = V.constructLops();
        if (needPartV) {
            // requires partitioning
            lV = new DataPartition(lV, DataType.MATRIX, ValueType.DOUBLE, (m2Size > OptimizerUtils.getLocalMemBudget()) ? ExecType.MR : ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
            lV.getOutputParameters().setDimensions(V.getDim1(), V.getDim2(), getRowsInBlock(), getColsInBlock(), V.getNnz());
            setLineNumbers(lV);
        }
        // map-side wsloss always with broadcast
        Lop wsloss = new WeightedSquaredLoss(X.constructLops(), lU, lV, W.constructLops(), DataType.MATRIX, ValueType.DOUBLE, wtype, ExecType.MR);
        wsloss.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
        setLineNumbers(wsloss);
        Group grp = new Group(wsloss, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
        grp.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
        setLineNumbers(grp);
        Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(AggOp.SUM), DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
        // aggregation uses kahanSum
        agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
        agg1.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
        setLineNumbers(agg1);
        UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
        unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(unary1);
        setLops(unary1);
    } else // general case
    {
        // MR operator selection part 2
        boolean cacheU = !FORCE_REPLICATION && (m1Size < OptimizerUtils.getRemoteMemBudgetReduce());
        boolean cacheV = !FORCE_REPLICATION && ((!cacheU && m2Size < OptimizerUtils.getRemoteMemBudgetReduce()) || (cacheU && m1Size + m2Size < OptimizerUtils.getRemoteMemBudgetReduce()));
        Group grpX = new Group(X.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
        grpX.getOutputParameters().setDimensions(X.getDim1(), X.getDim2(), X.getRowsInBlock(), X.getColsInBlock(), -1);
        setLineNumbers(grpX);
        Lop grpW = W.constructLops();
        if (grpW.getDataType() == DataType.MATRIX) {
            grpW = new Group(W.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
            grpW.getOutputParameters().setDimensions(W.getDim1(), W.getDim2(), W.getRowsInBlock(), W.getColsInBlock(), -1);
            setLineNumbers(grpW);
        }
        Lop lU = constructLeftFactorMRLop(U, V, cacheU, m1Size);
        Lop lV = constructRightFactorMRLop(U, V, cacheV, m2Size);
        // reduce-side wsloss w/ or without broadcast
        Lop wsloss = new WeightedSquaredLossR(grpX, lU, lV, grpW, DataType.MATRIX, ValueType.DOUBLE, wtype, cacheU, cacheV, ExecType.MR);
        wsloss.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
        setLineNumbers(wsloss);
        Group grp = new Group(wsloss, Group.OperationTypes.Sort, DataType.MATRIX, ValueType.DOUBLE);
        grp.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
        setLineNumbers(grp);
        Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(AggOp.SUM), DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
        // aggregation uses kahanSum
        agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
        agg1.getOutputParameters().setDimensions(1, 1, X.getRowsInBlock(), X.getColsInBlock(), -1);
        setLineNumbers(agg1);
        UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
        unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(unary1);
        setLops(unary1);
    }
}