Examples with DataGenOp org.apache.sysml.hops.DataGenOp used on opensource projects

Example 36 with DataGenOp

use of org.apache.sysml.hops.DataGenOp in project systemml by apache.

the class Recompiler method rUpdateStatistics.

public static void rUpdateStatistics(Hop hop, LocalVariableMap vars) {
    if (hop.isVisited())
        return;
    // recursively process children
    if (hop.getInput() != null)
        for (Hop c : hop.getInput()) rUpdateStatistics(c, vars);
    boolean updatedSizeExpr = false;
    // (with awareness not to override persistent reads to an existing name)
    if (hop instanceof DataOp && ((DataOp) hop).getDataOpType() != DataOpTypes.PERSISTENTREAD) {
        DataOp d = (DataOp) hop;
        String varName = d.getName();
        if (vars.keySet().contains(varName)) {
            Data dat = vars.get(varName);
            if (dat instanceof MatrixObject) {
                MatrixObject mo = (MatrixObject) dat;
                d.setDim1(mo.getNumRows());
                d.setDim2(mo.getNumColumns());
                d.setNnz(mo.getNnz());
            } else if (dat instanceof FrameObject) {
                FrameObject fo = (FrameObject) dat;
                d.setDim1(fo.getNumRows());
                d.setDim2(fo.getNumColumns());
            }
        }
    } else // special case for persistent reads with unknown size (read-after-write)
    if (hop instanceof DataOp && ((DataOp) hop).getDataOpType() == DataOpTypes.PERSISTENTREAD && !hop.dimsKnown() && ((DataOp) hop).getInputFormatType() != FileFormatTypes.CSV && !ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_READ_WRITE_METADATA)) {
        // update hop with read meta data
        DataOp dop = (DataOp) hop;
        tryReadMetaDataFileMatrixCharacteristics(dop);
    } else // update size expression for rand/seq according to symbol table entries
    if (hop instanceof DataGenOp) {
        DataGenOp d = (DataGenOp) hop;
        HashMap<String, Integer> params = d.getParamIndexMap();
        if (d.getOp() == DataGenMethod.RAND || d.getOp() == DataGenMethod.SINIT || d.getOp() == DataGenMethod.SAMPLE) {
            boolean initUnknown = !d.dimsKnown();
            int ix1 = params.get(DataExpression.RAND_ROWS);
            int ix2 = params.get(DataExpression.RAND_COLS);
            // update rows/cols by evaluating simple expression of literals, nrow, ncol, scalars, binaryops
            HashMap<Long, Long> memo = new HashMap<>();
            d.refreshRowsParameterInformation(d.getInput().get(ix1), vars, memo);
            d.refreshColsParameterInformation(d.getInput().get(ix2), vars, memo);
            updatedSizeExpr = initUnknown & d.dimsKnown();
        } else if (d.getOp() == DataGenMethod.SEQ) {
            boolean initUnknown = !d.dimsKnown();
            int ix1 = params.get(Statement.SEQ_FROM);
            int ix2 = params.get(Statement.SEQ_TO);
            int ix3 = params.get(Statement.SEQ_INCR);
            HashMap<Long, Double> memo = new HashMap<>();
            double from = d.computeBoundsInformation(d.getInput().get(ix1), vars, memo);
            double to = d.computeBoundsInformation(d.getInput().get(ix2), vars, memo);
            double incr = d.computeBoundsInformation(d.getInput().get(ix3), vars, memo);
            // special case increment
            if (from != Double.MAX_VALUE && to != Double.MAX_VALUE) {
                incr *= ((from > to && incr > 0) || (from < to && incr < 0)) ? -1.0 : 1.0;
            }
            if (from != Double.MAX_VALUE && to != Double.MAX_VALUE && incr != Double.MAX_VALUE) {
                d.setDim1(UtilFunctions.getSeqLength(from, to, incr));
                d.setDim2(1);
                d.setIncrementValue(incr);
            }
            updatedSizeExpr = initUnknown & d.dimsKnown();
        } else {
            throw new DMLRuntimeException("Unexpected data generation method: " + d.getOp());
        }
    } else // update size expression for reshape according to symbol table entries
    if (hop instanceof ReorgOp && ((ReorgOp) (hop)).getOp() == Hop.ReOrgOp.RESHAPE) {
        ReorgOp d = (ReorgOp) hop;
        boolean initUnknown = !d.dimsKnown();
        HashMap<Long, Long> memo = new HashMap<>();
        d.refreshRowsParameterInformation(d.getInput().get(1), vars, memo);
        d.refreshColsParameterInformation(d.getInput().get(2), vars, memo);
        updatedSizeExpr = initUnknown & d.dimsKnown();
    } else // update size expression for indexing according to symbol table entries
    if (hop instanceof IndexingOp) {
        IndexingOp iop = (IndexingOp) hop;
        // inpRowL
        Hop input2 = iop.getInput().get(1);
        // inpRowU
        Hop input3 = iop.getInput().get(2);
        // inpColL
        Hop input4 = iop.getInput().get(3);
        // inpColU
        Hop input5 = iop.getInput().get(4);
        boolean initUnknown = !iop.dimsKnown();
        HashMap<Long, Double> memo = new HashMap<>();
        double rl = iop.computeBoundsInformation(input2, vars, memo);
        double ru = iop.computeBoundsInformation(input3, vars, memo);
        double cl = iop.computeBoundsInformation(input4, vars, memo);
        double cu = iop.computeBoundsInformation(input5, vars, memo);
        if (rl != Double.MAX_VALUE && ru != Double.MAX_VALUE)
            iop.setDim1((long) (ru - rl + 1));
        if (cl != Double.MAX_VALUE && cu != Double.MAX_VALUE)
            iop.setDim2((long) (cu - cl + 1));
        updatedSizeExpr = initUnknown & iop.dimsKnown();
    }
    // without overwriting inferred size expressions
    if (!updatedSizeExpr) {
        hop.refreshSizeInformation();
    }
    hop.setVisited();
}

Example 37 with DataGenOp

use of org.apache.sysml.hops.DataGenOp in project systemml by apache.

the class HopRewriteUtils method createDataGenOp.

public static Hop createDataGenOp(Hop input, double value) {
    Hop rows = input.rowsKnown() ? new LiteralOp(input.getDim1()) : new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, input);
    Hop cols = input.colsKnown() ? new LiteralOp(input.getDim2()) : new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, OpOp1.NCOL, input);
    Hop val = new LiteralOp(value);
    HashMap<String, Hop> params = new HashMap<>();
    params.put(DataExpression.RAND_ROWS, rows);
    params.put(DataExpression.RAND_COLS, cols);
    params.put(DataExpression.RAND_MIN, val);
    params.put(DataExpression.RAND_MAX, val);
    params.put(DataExpression.RAND_PDF, new LiteralOp(DataExpression.RAND_PDF_UNIFORM));
    params.put(DataExpression.RAND_LAMBDA, new LiteralOp(-1.0));
    params.put(DataExpression.RAND_SPARSITY, new LiteralOp(1.0));
    params.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED));
    // note internal refresh size information
    Hop datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params);
    datagen.setOutputBlocksizes(input.getRowsInBlock(), input.getColsInBlock());
    copyLineNumbers(input, datagen);
    if (value == 0)
        datagen.setNnz(0);
    return datagen;
}

Example 38 with DataGenOp

use of org.apache.sysml.hops.DataGenOp in project systemml by apache.

the class HopRewriteUtils method createDataGenOp.

public static Hop createDataGenOp(Hop rowInput, boolean tRowInput, Hop colInput, boolean tColInput, double value) {
    long nrow = tRowInput ? rowInput.getDim2() : rowInput.getDim1();
    long ncol = tColInput ? colInput.getDim1() : rowInput.getDim2();
    Hop rows = (nrow >= 0) ? new LiteralOp(nrow) : new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, tRowInput ? OpOp1.NCOL : OpOp1.NROW, rowInput);
    Hop cols = (ncol >= 0) ? new LiteralOp(ncol) : new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, tColInput ? OpOp1.NROW : OpOp1.NCOL, colInput);
    Hop val = new LiteralOp(value);
    HashMap<String, Hop> params = new HashMap<>();
    params.put(DataExpression.RAND_ROWS, rows);
    params.put(DataExpression.RAND_COLS, cols);
    params.put(DataExpression.RAND_MIN, val);
    params.put(DataExpression.RAND_MAX, val);
    params.put(DataExpression.RAND_PDF, new LiteralOp(DataExpression.RAND_PDF_UNIFORM));
    params.put(DataExpression.RAND_LAMBDA, new LiteralOp(-1.0));
    params.put(DataExpression.RAND_SPARSITY, new LiteralOp(1.0));
    params.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED));
    // note internal refresh size information
    Hop datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params);
    datagen.setOutputBlocksizes(rowInput.getRowsInBlock(), colInput.getColsInBlock());
    copyLineNumbers(rowInput, datagen);
    if (value == 0)
        datagen.setNnz(0);
    return datagen;
}

Example 39 with DataGenOp

use of org.apache.sysml.hops.DataGenOp in project systemml by apache.

the class RewriteAlgebraicSimplificationDynamic method simplifyIdentityRepMatrixMult.

private static Hop simplifyIdentityRepMatrixMult(Hop parent, Hop hi, int pos) {
    if (// X%*%Y -> X, if y is matrix(1,1,1)
    HopRewriteUtils.isMatrixMultiply(hi)) {
        Hop left = hi.getInput().get(0);
        Hop right = hi.getInput().get(1);
        // X %*% y -> X
        if (// scalar right
        HopRewriteUtils.isDimsKnown(right) && right.getDim1() == 1 && right.getDim2() == 1 && right instanceof DataGenOp && ((DataGenOp) right).getOp() == DataGenMethod.RAND && // matrix(1,)
        ((DataGenOp) right).hasConstantValue(1.0)) {
            HopRewriteUtils.replaceChildReference(parent, hi, left, pos);
            hi = left;
            LOG.debug("Applied simplifyIdentiyMatrixMult");
        }
    }
    return hi;
}

Example 40 with DataGenOp

use of org.apache.sysml.hops.DataGenOp in project systemml by apache.

the class HopRewriteUtils method createDataGenOpByVal.

public static Hop createDataGenOpByVal(ArrayList<LiteralOp> values, long rows, long cols) {
    StringBuilder sb = new StringBuilder();
    for (LiteralOp lit : values) {
        if (sb.length() > 0)
            sb.append(StringInitCPInstruction.DELIM);
        sb.append(lit.getStringValue());
    }
    LiteralOp str = new LiteralOp(sb.toString());
    HashMap<String, Hop> params = new HashMap<>();
    params.put(DataExpression.RAND_ROWS, new LiteralOp(rows));
    params.put(DataExpression.RAND_COLS, new LiteralOp(cols));
    params.put(DataExpression.RAND_MIN, str);
    params.put(DataExpression.RAND_MAX, str);
    params.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED));
    Hop datagen = new DataGenOp(DataGenMethod.SINIT, new DataIdentifier("tmp"), params);
    int blksz = ConfigurationManager.getBlocksize();
    datagen.setOutputBlocksizes(blksz, blksz);
    copyLineNumbers(values.get(0), datagen);
    return datagen;
}