Examples with DMLRuntimeException org.apache.sysml.runtime.DMLRuntimeException used on opensource projects

Example 36 with DMLRuntimeException

use of org.apache.sysml.runtime.DMLRuntimeException in project incubator-systemml by apache.

the class LibMatrixDNNHelper method getConv2dWorkers.

/**
	 * Factory method that returns list of callable tasks for performing conv2d
	 * 
	 * @param params convolution parameters
	 * @return list of callable tasks for performing conv2d
	 * @throws DMLRuntimeException if error occurs
	 */
public static ArrayList<Callable<Long>> getConv2dWorkers(ConvolutionParameters params) throws DMLRuntimeException {
    ArrayList<Callable<Long>> ret = new ArrayList<Callable<Long>>();
    // Try to create as many tasks as threads. 
    // Creating more tasks will help in tail, but would have additional overhead of maintaining the intermediate
    // data structures such as im2col blocks.
    int k = OptimizerUtils.getConstrainedNumThreads(params.numThreads);
    int taskSize = (int) (Math.ceil((double) params.N / k));
    // TODO: Decide here based on params whether to use LoopedIm2ColConv2dAllChannels or LoopedIm2ColConv2dOneChannel
    // For now, let's stick to the existing approach of converting [1, CHW] to [CRS, PQ] as it allows matrix multiplication large enough matrix.
    boolean allChannels = true;
    ArrayList<MatrixBlock> filters = null;
    if (!allChannels) {
        filters = splitFilter(params);
    }
    boolean isEmptyDenseInput = !params.input1.isInSparseFormat() && params.input1.denseBlock == null;
    for (int i = 0; i * taskSize < params.N; i++) {
        if (LibMatrixDNN.isEligibleForConv2dSparse(params))
            ret.add(new LibMatrixDNNConv2dHelper.SparseNativeConv2d(i * taskSize, Math.min((i + 1) * taskSize, params.N), params));
        else if (!isEmptyDenseInput && allChannels)
            ret.add(new LibMatrixDNNConv2dHelper.LoopedIm2ColConv2dAllChannels(i * taskSize, Math.min((i + 1) * taskSize, params.N), params));
        else if (!isEmptyDenseInput && !allChannels)
            ret.add(new LibMatrixDNNConv2dHelper.LoopedIm2ColConv2dOneChannel(i * taskSize, Math.min((i + 1) * taskSize, params.N), params, filters));
        else
            throw new DMLRuntimeException("Unsupported operator");
    }
    return ret;
}

Example 37 with DMLRuntimeException

use of org.apache.sysml.runtime.DMLRuntimeException in project incubator-systemml by apache.

the class LibMatrixDNNHelper method doCol2imOverSingleImage.

// ------------------------------------------------------------------------------------------------------
// Since col2im always operates on intermediate generated as part of matmult, it is not clear which operator to select apriori.
// Therefore, it is provided as utility function rather than an operator (like im2col or rotate180)
//Converts input: PQ X CRS matrix and writes to 1 X CHW
static void doCol2imOverSingleImage(int outputN, MatrixBlock input, ConvolutionParameters params) throws DMLRuntimeException {
    if (input.rlen != params.P * params.Q || input.clen != params.C * params.R * params.S) {
        throw new DMLRuntimeException("Incorrect input dimensions");
    }
    double[] outputArray = null;
    if (!params.output.isInSparseFormat())
        outputArray = params.output.getDenseBlock();
    else {
        throw new DMLRuntimeException("Only dense output is implemented");
    }
    if (!input.isInSparseFormat()) {
        double[] inputArray = input.getDenseBlock();
        doCol2IMDenseInput(0, outputN, inputArray, outputArray, params);
    } else {
        if (!input.isEmptyBlock()) {
            int[] tensorIndexes = new int[3];
            for (int i = 0; i < input.getNumRows(); i++) {
                if (!input.sparseBlock.isEmpty(i)) {
                    computeTensorIndexes(i, tensorIndexes, params.P, params.Q);
                    int p = tensorIndexes[1];
                    int q = tensorIndexes[2];
                    if (tensorIndexes[0] != 0)
                        throw new DMLRuntimeException("Incorrect tensor indexes: " + tensorIndexes[0] + " != 0 <" + p + " " + q + " " + tensorIndexes[0] + params.P + " " + params.Q + ">");
                    int apos = input.sparseBlock.pos(i);
                    int alen = input.sparseBlock.size(i);
                    int[] aix = input.sparseBlock.indexes(i);
                    double[] avals = input.sparseBlock.values(i);
                    for (int j = apos; j < apos + alen; j++) {
                        computeTensorIndexes(aix[j], tensorIndexes, params.R, params.S);
                        int c = tensorIndexes[0];
                        int r = tensorIndexes[1];
                        int s = tensorIndexes[2];
                        int h = p * params.stride_h + r - params.pad_h;
                        int w = q * params.stride_w + s - params.pad_w;
                        if (h >= 0 && h < params.H && w >= 0 && w < params.W) {
                            int outIndex = outputN * params.C * params.H * params.W + c * params.H * params.W + h * params.W + w;
                            outputArray[outIndex] += avals[j];
                        }
                    }
                }
            }
        }
    }
}

Example 38 with DMLRuntimeException

use of org.apache.sysml.runtime.DMLRuntimeException in project incubator-systemml by apache.

the class LibMatrixDatagen method generateRandomMatrix.

/**
	 * Function to generate a matrix of random numbers. This is invoked both
	 * from CP as well as from MR. In case of CP, it generates an entire matrix
	 * block-by-block. A <code>bigrand</code> is passed so that block-level
	 * seeds are generated internally. In case of MR, it generates a single
	 * block for given block-level seed <code>bSeed</code>.
	 * 
	 * When pdf="uniform", cell values are drawn from uniform distribution in
	 * range <code>[min,max]</code>.
	 * 
	 * When pdf="normal", cell values are drawn from standard normal
	 * distribution N(0,1). The range of generated values will always be
	 * (-Inf,+Inf).
	 * 
	 * 
     * @param out output matrix block
     * @param rgen random matrix generator
     * @param nnzInBlocks number of non-zeros in blocks
     * @param bigrand Well1024a pseudo-random number generator
     * @param bSeed seed for random generator
     * @param k ?
     * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
public static void generateRandomMatrix(MatrixBlock out, RandomMatrixGenerator rgen, LongStream nnzInBlocks, Well1024a bigrand, long bSeed, int k) throws DMLRuntimeException {
    int rows = rgen._rows;
    int cols = rgen._cols;
    int rpb = rgen._rowsPerBlock;
    int cpb = rgen._colsPerBlock;
    double sparsity = rgen._sparsity;
    //sanity check valid dimensions and sparsity
    checkMatrixDimensionsAndSparsity(rows, cols, sparsity);
    /*
		 * Setup min and max for distributions other than "uniform". Min and Max
		 * are set up in such a way that the usual logic of
		 * (max-min)*prng.nextDouble() is still valid. This is done primarily to
		 * share the same code across different distributions.
		 */
    double min = rgen._pdf == RandomMatrixGenerator.PDF.UNIFORM ? rgen._min : 0;
    double max = rgen._pdf == RandomMatrixGenerator.PDF.UNIFORM ? rgen._max : 1;
    //determine the sparsity of output matrix (multi-threaded always invoked from CP):
    //estimated NNZ is for entire matrix (nnz=0, if 0 initialized)
    final long estnnz = ((min == 0.0 && max == 0.0) ? 0 : (long) (sparsity * rows * cols));
    boolean lsparse = MatrixBlock.evalSparseFormatInMemory(rows, cols, estnnz);
    //fallback to sequential if single rowblock or too few cells or if MatrixBlock is not thread safe
    if (k <= 1 || (rows <= rpb && lsparse) || (long) rows * cols < PAR_NUMCELL_THRESHOLD || !MatrixBlock.isThreadSafe(lsparse)) {
        generateRandomMatrix(out, rgen, nnzInBlocks, bigrand, bSeed);
        return;
    }
    //special case shortcuts for efficiency
    if (rgen._pdf == RandomMatrixGenerator.PDF.UNIFORM) {
        if (min == 0.0 && max == 0.0) {
            //all zeros
            out.reset(rows, cols, false);
            return;
        } else if (sparsity == 1.0d && min == max) {
            //equal values
            out.reset(rows, cols, min);
            return;
        }
    }
    //allocate memory
    //note: individual sparse rows are allocated on demand,
    //for consistency with memory estimates and prevent OOMs.
    out.reset(rows, cols, lsparse);
    if (out.sparse)
        out.allocateSparseRowsBlock();
    else
        out.allocateDenseBlock();
    int nrb = (int) Math.ceil((double) rows / rpb);
    int ncb = (int) Math.ceil((double) cols / cpb);
    //default: parallelization over row blocks, fallback to parallelization
    //over column blocks if possible and necessary (higher degree of par)
    boolean parcol = (!out.sparse && nrb < k && ncb > nrb);
    int parnb = parcol ? ncb : nrb;
    //generate seeds independent of parallelizations
    long[] seeds = generateSeedsForCP(bigrand, nrb, ncb);
    // collect nnz stream for multiple consumptions
    long[] lnnzInBlocks = nnzInBlocks.toArray();
    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<RandTask> tasks = new ArrayList<RandTask>();
        int blklen = ((int) (Math.ceil((double) parnb / k)));
        for (int i = 0; i < k & i * blklen < parnb; i++) {
            int rl = parcol ? 0 : i * blklen;
            int ru = parcol ? nrb : Math.min((i + 1) * blklen, parnb);
            int cl = parcol ? i * blklen : 0;
            int cu = parcol ? Math.min((i + 1) * blklen, parnb) : ncb;
            long[] lseeds = sliceSeedsForCP(seeds, rl, ru, cl, cu, nrb, ncb);
            tasks.add(new RandTask(rl, ru, cl, cu, out, rgen, lnnzInBlocks, bSeed, lseeds));
        }
        List<Future<Object>> ret = pool.invokeAll(tasks);
        pool.shutdown();
        //exception propagation in case not all tasks successful
        for (Future<Object> rc : ret) rc.get();
    } catch (Exception e) {
        throw new DMLRuntimeException(e);
    }
    out.recomputeNonZeros();
}

Example 39 with DMLRuntimeException

use of org.apache.sysml.runtime.DMLRuntimeException in project incubator-systemml by apache.

the class LibMatrixDNN method maxpoolingBackward.

/**
	 * This method computes the backpropogation errors for previous layer of maxpooling operation
	 * 
	 * @param input input matrix
	 * @param dout dout matrix
	 * @param outputBlock output matrix
	 * @param params convolution parameters
	 * @param performReluBackward perform ReLU backward
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
public static void maxpoolingBackward(MatrixBlock input, MatrixBlock dout, MatrixBlock outputBlock, ConvolutionParameters params, boolean performReluBackward) throws DMLRuntimeException {
    params.input1 = input;
    params.input2 = dout;
    params.output = outputBlock;
    if (input.getNumColumns() != params.C * params.H * params.W || input.getNumRows() != params.N) {
        throw new DMLRuntimeException("Incorrect input dimensions in maxpooling_backward:" + input.getNumRows() + " " + input.getNumColumns() + " " + params.N + " " + params.K * params.P * params.Q);
    }
    if (dout.getNumColumns() != params.C * params.P * params.Q || dout.getNumRows() != params.N) {
        throw new DMLRuntimeException("Incorrect dout dimensions in maxpooling_backward:" + input.getNumRows() + " " + input.getNumColumns() + " " + params.N + " " + params.K * params.P * params.Q);
    }
    if (DMLScript.STATISTICS && DISPLAY_STATISTICS) {
        if (input.isInSparseFormat() || dout.isInSparseFormat()) {
            maxPoolBwdSparseCount.addAndGet(1);
        } else {
            maxPoolBwdDenseCount.addAndGet(1);
        }
    }
    if (params.output.isInSparseFormat())
        throw new DMLRuntimeException("Sparse maxpooling_backward is not supported");
    fillIndexesArray(params);
    execute(LibMatrixDNNHelper.getMaxPoolingBackwardWorkers(params, performReluBackward), params);
    //post-processing: maintain nnz 
    outputBlock.recomputeNonZeros();
}

Example 40 with DMLRuntimeException

use of org.apache.sysml.runtime.DMLRuntimeException in project incubator-systemml by apache.

the class Connection method convertToMatrix.

/**
 * Converts an input stream of a string matrix in csv or textcell format
 * into a matrix block.
 *
 * @param input InputStream to a string matrix in csv or textcell format
 * @param rows number of rows in the matrix
 * @param cols number of columns in the matrix
 * @param format input format of the given stream
 * @return matrix as a matrix block
 * @throws IOException if IOException occurs
 */
public MatrixBlock convertToMatrix(InputStream input, int rows, int cols, String format) throws IOException {
    MatrixBlock ret = null;
    // sanity check input format
    if (!(DataExpression.FORMAT_TYPE_VALUE_TEXT.equals(format) || DataExpression.FORMAT_TYPE_VALUE_MATRIXMARKET.equals(format) || DataExpression.FORMAT_TYPE_VALUE_CSV.equals(format))) {
        throw new IOException("Invalid input format (expected: csv, text or mm): " + format);
    }
    setLocalConfigs();
    try {
        // read input matrix
        InputInfo iinfo = DataExpression.FORMAT_TYPE_VALUE_CSV.equals(format) ? InputInfo.CSVInputInfo : InputInfo.TextCellInputInfo;
        MatrixReader reader = MatrixReaderFactory.createMatrixReader(iinfo);
        int blksz = ConfigurationManager.getBlocksize();
        ret = reader.readMatrixFromInputStream(input, rows, cols, blksz, blksz, (long) rows * cols);
    } catch (DMLRuntimeException rex) {
        throw new IOException(rex);
    }
    return ret;
}