Examples with LopsException org.apache.sysml.lops.LopsException used on opensource projects

Example 16 with LopsException

use of org.apache.sysml.lops.LopsException in project systemml by apache.

the class Hop method constructAndSetCheckpointLopIfRequired.

private void constructAndSetCheckpointLopIfRequired() {
    // determine execution type
    ExecType et = ExecType.CP;
    if (OptimizerUtils.isSparkExecutionMode() && getDataType() != DataType.SCALAR) {
        // (2) avoid unnecessary creation of spark context (incl executors)
        if ((OptimizerUtils.isHybridExecutionMode() && hasValidCPDimsAndSize() && !OptimizerUtils.exceedsCachingThreshold(getDim2(), _outputMemEstimate)) || _etypeForced == ExecType.CP) {
            et = ExecType.CP;
        } else // default case
        {
            et = ExecType.SPARK;
        }
    }
    // add checkpoint lop to output if required
    if (_requiresCheckpoint && et != ExecType.CP) {
        try {
            // investigate need for serialized storage of large sparse matrices
            // (compile- instead of runtime-level for better debugging)
            boolean serializedStorage = false;
            if (getDataType() == DataType.MATRIX && dimsKnown(true)) {
                double matrixPSize = OptimizerUtils.estimatePartitionedSizeExactSparsity(_dim1, _dim2, _rows_in_block, _cols_in_block, _nnz);
                double dataCache = SparkExecutionContext.getDataMemoryBudget(true, true);
                serializedStorage = MatrixBlock.evalSparseFormatInMemory(_dim1, _dim2, _nnz) && // sparse in-memory does not fit in agg mem
                matrixPSize > dataCache && (OptimizerUtils.getSparsity(_dim1, _dim2, _nnz) < MatrixBlock.ULTRA_SPARSITY_TURN_POINT || // ultra-sparse or sparse w/o csr
                !Checkpoint.CHECKPOINT_SPARSE_CSR);
            } else if (!dimsKnown(true)) {
                setRequiresRecompile();
            }
            // construct checkpoint w/ right storage level
            Lop input = getLops();
            Lop chkpoint = new Checkpoint(input, getDataType(), getValueType(), serializedStorage ? Checkpoint.getSerializeStorageLevelString() : Checkpoint.getDefaultStorageLevelString());
            setOutputDimensions(chkpoint);
            setLineNumbers(chkpoint);
            setLops(chkpoint);
        } catch (LopsException ex) {
            throw new HopsException(ex);
        }
    }
}

Example 17 with LopsException

use of org.apache.sysml.lops.LopsException in project systemml by apache.

the class LiteralOp method constructLops.

@Override
public Lop constructLops() {
    // return already created lops
    if (getLops() != null)
        return getLops();
    try {
        Lop l = null;
        switch(getValueType()) {
            case DOUBLE:
                l = Data.createLiteralLop(ValueType.DOUBLE, Double.toString(value_double));
                break;
            case BOOLEAN:
                l = Data.createLiteralLop(ValueType.BOOLEAN, Boolean.toString(value_boolean));
                break;
            case STRING:
                l = Data.createLiteralLop(ValueType.STRING, value_string);
                break;
            case INT:
                l = Data.createLiteralLop(ValueType.INT, Long.toString(value_long));
                break;
            default:
                throw new HopsException(this.printErrorLocation() + "unexpected value type constructing lops for LiteralOp.\n");
        }
        l.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(l);
        setLops(l);
    } catch (LopsException e) {
        throw new HopsException(e);
    }
    return getLops();
}

Example 18 with LopsException

use of org.apache.sysml.lops.LopsException in project incubator-systemml by apache.

the class MultipleOp method constructLops.

/**
	 * Construct the corresponding Lops for this Hop
	 */
@Override
public Lop constructLops() throws HopsException, LopsException {
    // reuse existing lop
    if (getLops() != null)
        return getLops();
    try {
        ArrayList<Hop> inHops = getInput();
        Lop[] inLops = new Lop[inHops.size()];
        for (int i = 0; i < inHops.size(); i++) {
            Hop inHop = inHops.get(i);
            Lop inLop = inHop.constructLops();
            inLops[i] = inLop;
        }
        MultipleCP.OperationType opType = MultipleOperandOperationHopTypeToLopType.get(multipleOperandOperation);
        if (opType == null) {
            throw new HopsException("Unknown MultipleCP Lop operation type for MultipleOperandOperation Hop type '" + multipleOperandOperation + "'");
        }
        MultipleCP multipleCPLop = new MultipleCP(opType, getDataType(), getValueType(), inLops);
        setOutputDimensions(multipleCPLop);
        setLineNumbers(multipleCPLop);
        setLops(multipleCPLop);
    } catch (Exception e) {
        throw new HopsException(this.printErrorLocation() + "error constructing Lops for MultipleOp Hop -- \n ", e);
    }
    // add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 19 with LopsException

use of org.apache.sysml.lops.LopsException in project incubator-systemml by apache.

the class Dag method getRecordReaderInstructions.

/**
 * Method to get record reader instructions for a MR job.
 *
 * @param node low-level operator
 * @param execNodes list of exec nodes
 * @param inputStrings list of input strings
 * @param recordReaderInstructions list of record reader instructions
 * @param nodeIndexMapping node index mapping
 * @param start_index start index
 * @param inputLabels list of input labels
 * @param inputLops list of input lops
 * @param MRJobLineNumbers MR job line numbers
 * @return -1 if problem
 */
private static int getRecordReaderInstructions(Lop node, ArrayList<Lop> execNodes, ArrayList<String> inputStrings, ArrayList<String> recordReaderInstructions, HashMap<Lop, Integer> nodeIndexMapping, int[] start_index, ArrayList<String> inputLabels, ArrayList<Lop> inputLops, ArrayList<Integer> MRJobLineNumbers) {
    // if input source, return index
    if (nodeIndexMapping.containsKey(node))
        return nodeIndexMapping.get(node);
    // not input source and not in exec nodes, then return.
    if (!execNodes.contains(node))
        return -1;
    ArrayList<Integer> inputIndices = new ArrayList<>();
    int max_input_index = -1;
    // get mapper instructions
    for (int i = 0; i < node.getInputs().size(); i++) {
        // recurse
        Lop childNode = node.getInputs().get(i);
        int ret_val = getRecordReaderInstructions(childNode, execNodes, inputStrings, recordReaderInstructions, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers);
        inputIndices.add(ret_val);
        if (ret_val > max_input_index) {
            max_input_index = ret_val;
        // child_for_max_input_index = childNode;
        }
    }
    // instructions
    if ((node.getExecLocation() == ExecLocation.RecordReader)) {
        int output_index = max_input_index;
        // cannot reuse index if this is true
        // need to add better indexing schemes
        output_index = start_index[0];
        start_index[0]++;
        nodeIndexMapping.put(node, output_index);
        // only Ranagepick lop can contribute to labels
        if (node.getType() == Type.PickValues) {
            PickByCount pbc = (PickByCount) node;
            if (pbc.getOperationType() == PickByCount.OperationTypes.RANGEPICK) {
                // always the second input is a scalar
                int scalarIndex = 1;
                // if data lop not a literal -- add label
                if (node.getInputs().get(scalarIndex).getExecLocation() == ExecLocation.Data && !((Data) (node.getInputs().get(scalarIndex))).isLiteral()) {
                    inputLabels.add(node.getInputs().get(scalarIndex).getOutputParameters().getLabel());
                    inputLops.add(node.getInputs().get(scalarIndex));
                }
                // if not data lop, then this is an intermediate variable.
                if (node.getInputs().get(scalarIndex).getExecLocation() != ExecLocation.Data) {
                    inputLabels.add(node.getInputs().get(scalarIndex).getOutputParameters().getLabel());
                    inputLops.add(node.getInputs().get(scalarIndex));
                }
            }
        }
        // get recordreader instruction.
        if (node.getInputs().size() == 2) {
            recordReaderInstructions.add(node.getInstructions(inputIndices.get(0), inputIndices.get(1), output_index));
            if (DMLScript.ENABLE_DEBUG_MODE) {
                MRJobLineNumbers.add(node._beginLine);
            }
        } else
            throw new LopsException("Unexpected number of inputs while generating a RecordReader Instruction");
        return output_index;
    }
    return -1;
}

Example 20 with LopsException

use of org.apache.sysml.lops.LopsException in project incubator-systemml by apache.

the class Dag method generateInstructionsForInputVariables.

/**
 * Method to generate createvar instructions, which creates a new entry
 * in the symbol table. One instruction is generated for every LOP that is
 * 1) type Data and
 * 2) persistent and
 * 3) matrix and
 * 4) read
 *
 * Transient reads needn't be considered here since the previous program
 * block would already create appropriate entries in the symbol table.
 *
 * @param nodes_v list of nodes
 * @param inst list of instructions
 */
private static void generateInstructionsForInputVariables(ArrayList<Lop> nodes_v, ArrayList<Instruction> inst) {
    for (Lop n : nodes_v) {
        if (n.getExecLocation() == ExecLocation.Data && !((Data) n).isTransient() && ((Data) n).getOperationType() == OperationTypes.READ && (n.getDataType() == DataType.MATRIX || n.getDataType() == DataType.FRAME)) {
            if (!((Data) n).isLiteral()) {
                try {
                    String inst_string = n.getInstructions();
                    CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(inst_string);
                    currInstr.setLocation(n);
                    inst.add(currInstr);
                } catch (DMLRuntimeException e) {
                    throw new LopsException(n.printErrorLocation() + "error generating instructions from input variables in Dag -- \n", e);
                }
            }
        }
    }
}