Examples with DMLRuntimeException org.apache.sysml.runtime.DMLRuntimeException used on opensource projects

Example 96 with DMLRuntimeException

use of org.apache.sysml.runtime.DMLRuntimeException in project incubator-systemml by apache.

the class LibMatrixDNNHelper method getConv2dWorkers.

/**
	 * Factory method that returns list of callable tasks for performing conv2d
	 * 
	 * @param params convolution parameters
	 * @return list of callable tasks for performing conv2d
	 * @throws DMLRuntimeException if error occurs
	 */
public static ArrayList<Callable<Long>> getConv2dWorkers(ConvolutionParameters params) throws DMLRuntimeException {
    ArrayList<Callable<Long>> ret = new ArrayList<Callable<Long>>();
    // Try to create as many tasks as threads. 
    // Creating more tasks will help in tail, but would have additional overhead of maintaining the intermediate
    // data structures such as im2col blocks.
    int k = OptimizerUtils.getConstrainedNumThreads(params.numThreads);
    int taskSize = (int) (Math.ceil((double) params.N / k));
    // TODO: Decide here based on params whether to use LoopedIm2ColConv2dAllChannels or LoopedIm2ColConv2dOneChannel
    // For now, let's stick to the existing approach of converting [1, CHW] to [CRS, PQ] as it allows matrix multiplication large enough matrix.
    boolean allChannels = true;
    ArrayList<MatrixBlock> filters = null;
    if (!allChannels) {
        filters = splitFilter(params);
    }
    boolean isEmptyDenseInput = !params.input1.isInSparseFormat() && params.input1.denseBlock == null;
    for (int i = 0; i * taskSize < params.N; i++) {
        if (LibMatrixDNN.isEligibleForConv2dSparse(params))
            ret.add(new LibMatrixDNNConv2dHelper.SparseNativeConv2d(i * taskSize, Math.min((i + 1) * taskSize, params.N), params));
        else if (!isEmptyDenseInput && allChannels)
            ret.add(new LibMatrixDNNConv2dHelper.LoopedIm2ColConv2dAllChannels(i * taskSize, Math.min((i + 1) * taskSize, params.N), params));
        else if (!isEmptyDenseInput && !allChannels)
            ret.add(new LibMatrixDNNConv2dHelper.LoopedIm2ColConv2dOneChannel(i * taskSize, Math.min((i + 1) * taskSize, params.N), params, filters));
        else
            throw new DMLRuntimeException("Unsupported operator");
    }
    return ret;
}

Example 97 with DMLRuntimeException

use of org.apache.sysml.runtime.DMLRuntimeException in project incubator-systemml by apache.

the class LibMatrixDNNHelper method doCol2imOverSingleImage.

// ------------------------------------------------------------------------------------------------------
// Since col2im always operates on intermediate generated as part of matmult, it is not clear which operator to select apriori.
// Therefore, it is provided as utility function rather than an operator (like im2col or rotate180)
//Converts input: PQ X CRS matrix and writes to 1 X CHW
static void doCol2imOverSingleImage(int outputN, MatrixBlock input, ConvolutionParameters params) throws DMLRuntimeException {
    if (input.rlen != params.P * params.Q || input.clen != params.C * params.R * params.S) {
        throw new DMLRuntimeException("Incorrect input dimensions");
    }
    double[] outputArray = null;
    if (!params.output.isInSparseFormat())
        outputArray = params.output.getDenseBlock();
    else {
        throw new DMLRuntimeException("Only dense output is implemented");
    }
    if (!input.isInSparseFormat()) {
        double[] inputArray = input.getDenseBlock();
        doCol2IMDenseInput(0, outputN, inputArray, outputArray, params);
    } else {
        if (!input.isEmptyBlock()) {
            int[] tensorIndexes = new int[3];
            for (int i = 0; i < input.getNumRows(); i++) {
                if (!input.sparseBlock.isEmpty(i)) {
                    computeTensorIndexes(i, tensorIndexes, params.P, params.Q);
                    int p = tensorIndexes[1];
                    int q = tensorIndexes[2];
                    if (tensorIndexes[0] != 0)
                        throw new DMLRuntimeException("Incorrect tensor indexes: " + tensorIndexes[0] + " != 0 <" + p + " " + q + " " + tensorIndexes[0] + params.P + " " + params.Q + ">");
                    int apos = input.sparseBlock.pos(i);
                    int alen = input.sparseBlock.size(i);
                    int[] aix = input.sparseBlock.indexes(i);
                    double[] avals = input.sparseBlock.values(i);
                    for (int j = apos; j < apos + alen; j++) {
                        computeTensorIndexes(aix[j], tensorIndexes, params.R, params.S);
                        int c = tensorIndexes[0];
                        int r = tensorIndexes[1];
                        int s = tensorIndexes[2];
                        int h = p * params.stride_h + r - params.pad_h;
                        int w = q * params.stride_w + s - params.pad_w;
                        if (h >= 0 && h < params.H && w >= 0 && w < params.W) {
                            int outIndex = outputN * params.C * params.H * params.W + c * params.H * params.W + h * params.W + w;
                            outputArray[outIndex] += avals[j];
                        }
                    }
                }
            }
        }
    }
}

Example 98 with DMLRuntimeException

use of org.apache.sysml.runtime.DMLRuntimeException in project incubator-systemml by apache.

the class LibMatrixDatagen method generateSequence.

/**
	 * Method to generate a sequence according to the given parameters. The
	 * generated sequence is always in dense format.
	 * 
	 * Both end points specified <code>from</code> and <code>to</code> must be
	 * included in the generated sequence i.e., [from,to] both inclusive. Note
	 * that, <code>to</code> is included only if (to-from) is perfectly
	 * divisible by <code>incr</code>.
	 * 
	 * For example, seq(0,1,0.5) generates (0.0 0.5 1.0) 
	 *      whereas seq(0,1,0.6) generates (0.0 0.6) but not (0.0 0.6 1.0)
	 * 
	 * @param out output matrix block
	 * @param from lower end point
	 * @param to upper end point
	 * @param incr increment value
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
public static void generateSequence(MatrixBlock out, double from, double to, double incr) throws DMLRuntimeException {
    //check valid increment value
    if ((from > to && incr > 0) || incr == 0)
        throw new DMLRuntimeException("Wrong sequence increment: from=" + from + ", to=" + to + ", incr=" + incr);
    //prepare output matrix
    int rows = 1 + (int) Math.floor((to - from) / incr);
    // sequence vector always dense
    int cols = 1;
    out.reset(rows, cols, false);
    out.allocateDenseBlock();
    //compute sequence data
    double[] c = out.denseBlock;
    double cur = from;
    for (int i = 0; i < rows; i++) {
        c[i] = cur;
        cur += incr;
    }
    out.recomputeNonZeros();
}

Example 99 with DMLRuntimeException

use of org.apache.sysml.runtime.DMLRuntimeException in project incubator-systemml by apache.

the class LibMatrixDatagen method generateRandomMatrix.

/**
	 * Function to generate a matrix of random numbers. This is invoked both
	 * from CP as well as from MR. In case of CP, it generates an entire matrix
	 * block-by-block. A <code>bigrand</code> is passed so that block-level
	 * seeds are generated internally. In case of MR, it generates a single
	 * block for given block-level seed <code>bSeed</code>.
	 * 
	 * When pdf="uniform", cell values are drawn from uniform distribution in
	 * range <code>[min,max]</code>.
	 * 
	 * When pdf="normal", cell values are drawn from standard normal
	 * distribution N(0,1). The range of generated values will always be
	 * (-Inf,+Inf).
	 * 
	 * 
     * @param out output matrix block
     * @param rgen random matrix generator
     * @param nnzInBlocks number of non-zeros in blocks
     * @param bigrand Well1024a pseudo-random number generator
     * @param bSeed seed for random generator
     * @param k ?
     * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
public static void generateRandomMatrix(MatrixBlock out, RandomMatrixGenerator rgen, LongStream nnzInBlocks, Well1024a bigrand, long bSeed, int k) throws DMLRuntimeException {
    int rows = rgen._rows;
    int cols = rgen._cols;
    int rpb = rgen._rowsPerBlock;
    int cpb = rgen._colsPerBlock;
    double sparsity = rgen._sparsity;
    //sanity check valid dimensions and sparsity
    checkMatrixDimensionsAndSparsity(rows, cols, sparsity);
    /*
		 * Setup min and max for distributions other than "uniform". Min and Max
		 * are set up in such a way that the usual logic of
		 * (max-min)*prng.nextDouble() is still valid. This is done primarily to
		 * share the same code across different distributions.
		 */
    double min = rgen._pdf == RandomMatrixGenerator.PDF.UNIFORM ? rgen._min : 0;
    double max = rgen._pdf == RandomMatrixGenerator.PDF.UNIFORM ? rgen._max : 1;
    //determine the sparsity of output matrix (multi-threaded always invoked from CP):
    //estimated NNZ is for entire matrix (nnz=0, if 0 initialized)
    final long estnnz = ((min == 0.0 && max == 0.0) ? 0 : (long) (sparsity * rows * cols));
    boolean lsparse = MatrixBlock.evalSparseFormatInMemory(rows, cols, estnnz);
    //fallback to sequential if single rowblock or too few cells or if MatrixBlock is not thread safe
    if (k <= 1 || (rows <= rpb && lsparse) || (long) rows * cols < PAR_NUMCELL_THRESHOLD || !MatrixBlock.isThreadSafe(lsparse)) {
        generateRandomMatrix(out, rgen, nnzInBlocks, bigrand, bSeed);
        return;
    }
    //special case shortcuts for efficiency
    if (rgen._pdf == RandomMatrixGenerator.PDF.UNIFORM) {
        if (min == 0.0 && max == 0.0) {
            //all zeros
            out.reset(rows, cols, false);
            return;
        } else if (sparsity == 1.0d && min == max) {
            //equal values
            out.reset(rows, cols, min);
            return;
        }
    }
    //allocate memory
    //note: individual sparse rows are allocated on demand,
    //for consistency with memory estimates and prevent OOMs.
    out.reset(rows, cols, lsparse);
    if (out.sparse)
        out.allocateSparseRowsBlock();
    else
        out.allocateDenseBlock();
    int nrb = (int) Math.ceil((double) rows / rpb);
    int ncb = (int) Math.ceil((double) cols / cpb);
    //default: parallelization over row blocks, fallback to parallelization
    //over column blocks if possible and necessary (higher degree of par)
    boolean parcol = (!out.sparse && nrb < k && ncb > nrb);
    int parnb = parcol ? ncb : nrb;
    //generate seeds independent of parallelizations
    long[] seeds = generateSeedsForCP(bigrand, nrb, ncb);
    // collect nnz stream for multiple consumptions
    long[] lnnzInBlocks = nnzInBlocks.toArray();
    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<RandTask> tasks = new ArrayList<RandTask>();
        int blklen = ((int) (Math.ceil((double) parnb / k)));
        for (int i = 0; i < k & i * blklen < parnb; i++) {
            int rl = parcol ? 0 : i * blklen;
            int ru = parcol ? nrb : Math.min((i + 1) * blklen, parnb);
            int cl = parcol ? i * blklen : 0;
            int cu = parcol ? Math.min((i + 1) * blklen, parnb) : ncb;
            long[] lseeds = sliceSeedsForCP(seeds, rl, ru, cl, cu, nrb, ncb);
            tasks.add(new RandTask(rl, ru, cl, cu, out, rgen, lnnzInBlocks, bSeed, lseeds));
        }
        List<Future<Object>> ret = pool.invokeAll(tasks);
        pool.shutdown();
        //exception propagation in case not all tasks successful
        for (Future<Object> rc : ret) rc.get();
    } catch (Exception e) {
        throw new DMLRuntimeException(e);
    }
    out.recomputeNonZeros();
}

Example 100 with DMLRuntimeException

use of org.apache.sysml.runtime.DMLRuntimeException in project incubator-systemml by apache.

the class LibMatrixDatagen method createRandomMatrixGenerator.

public static RandomMatrixGenerator createRandomMatrixGenerator(String pdfStr, int r, int c, int rpb, int cpb, double sp, double min, double max, String distParams) throws DMLRuntimeException {
    RandomMatrixGenerator.PDF pdf = RandomMatrixGenerator.PDF.valueOf(pdfStr.toUpperCase());
    RandomMatrixGenerator rgen = null;
    switch(pdf) {
        case UNIFORM:
            rgen = new RandomMatrixGenerator(pdf, r, c, rpb, cpb, sp, min, max);
            break;
        case NORMAL:
            rgen = new RandomMatrixGenerator(pdf, r, c, rpb, cpb, sp);
            break;
        case POISSON:
            double mean = Double.NaN;
            try {
                mean = Double.parseDouble(distParams);
            } catch (NumberFormatException e) {
                throw new DMLRuntimeException("Failed to parse Poisson distribution parameter: " + distParams);
            }
            rgen = new RandomMatrixGenerator(pdf, r, c, rpb, cpb, sp, min, max, mean);
            break;
        default:
            throw new DMLRuntimeException("Unsupported probability distribution \"" + pdf + "\" in rand() -- it must be one of \"uniform\", \"normal\", or \"poisson\"");
    }
    return rgen;
}