Examples with DataIdentifier org.apache.sysml.parser.DataIdentifier used on opensource projects

Example 21 with DataIdentifier

use of org.apache.sysml.parser.DataIdentifier in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method getBlock2CellInstructions.

/**
 * Method to generate instructions to convert input matrices from block to
 * cell. We generate a GMR job here.
 *
 * @param inputParams list of data identifiers
 * @param unBlockedFileNames map of unblocked file names
 * @return list of instructions
 */
private ArrayList<Instruction> getBlock2CellInstructions(ArrayList<DataIdentifier> inputParams, HashMap<String, String> unBlockedFileNames) {
    ArrayList<Instruction> b2cinst = null;
    // list of input matrices
    ArrayList<DataIdentifier> matrices = new ArrayList<>();
    ArrayList<DataIdentifier> matricesNoReblock = new ArrayList<>();
    // find all inputs that are matrices
    for (int i = 0; i < inputParams.size(); i++) {
        if (inputParams.get(i).getDataType().isMatrix()) {
            if (_skipInReblock.contains(inputParams.get(i).getName()))
                matricesNoReblock.add(inputParams.get(i));
            else
                matrices.add(inputParams.get(i));
        }
    }
    if (!matrices.isEmpty()) {
        b2cinst = new ArrayList<>();
        MRJobInstruction gmrInst = new MRJobInstruction(JobType.GMR);
        TreeMap<Integer, ArrayList<String>> MRJobLineNumbers = null;
        if (DMLScript.ENABLE_DEBUG_MODE) {
            MRJobLineNumbers = new TreeMap<>();
        }
        String gmrStr = "";
        ArrayList<String> inLabels = new ArrayList<>();
        ArrayList<String> outLabels = new ArrayList<>();
        String[] outputs = new String[matrices.size()];
        byte[] resultIndex = new byte[matrices.size()];
        String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
        try {
            // create a GMR job that transforms each of these matrices from block to cell
            for (int i = 0; i < matrices.size(); i++) {
                inLabels.add(matrices.get(i).getName());
                outLabels.add(matrices.get(i).getName() + "_extFnInput");
                // (matrices.size()+i);
                resultIndex[i] = (byte) i;
                outputs[i] = scratchSpaceLoc + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR + _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Input";
                unBlockedFileNames.put(matrices.get(i).getName(), outputs[i]);
                if (DMLScript.ENABLE_DEBUG_MODE) {
                    // Create a dummy gmr instruction (FOR DEBUGGER)
                    gmrStr = "MR" + Lop.OPERAND_DELIMITOR + "gmr" + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize();
                    // Set MR gmr instruction line number (FOR DEBUGGER)
                    if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
                        MRJobLineNumbers.put(matrices.get(i).getBeginLine(), new ArrayList<String>());
                    }
                    MRJobLineNumbers.get(matrices.get(i).getBeginLine()).add(gmrStr);
                }
                // create metadata instructions to populate symbol table
                // with variables that hold unblocked matrices
                Instruction createInst = VariableCPInstruction.prepareCreateMatrixVariableInstruction(outLabels.get(i), outputs[i], false, OutputInfo.outputInfoToString(OutputInfo.TextCellOutputInfo));
                createInst.setLocation(matrices.get(i));
                b2cinst.add(createInst);
            }
            // Finally, generate GMR instruction that performs block2cell conversion
            gmrInst.setGMRInstructions(inLabels.toArray(new String[inLabels.size()]), "", "", "", "", outLabels.toArray(new String[outLabels.size()]), resultIndex, 0, 1);
            b2cinst.add(gmrInst);
            // generate instructions that rename the output variables of GMR job
            Instruction cpInst = null, rmInst = null;
            for (int i = 0; i < matrices.size(); i++) {
                cpInst = VariableCPInstruction.prepareCopyInstruction(outLabels.get(i), matrices.get(i).getName());
                rmInst = VariableCPInstruction.prepareRemoveInstruction(outLabels.get(i));
                cpInst.setLocation(matrices.get(i));
                rmInst.setLocation(matrices.get(i));
                b2cinst.add(cpInst);
                b2cinst.add(rmInst);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        // LOG instructions
        if (LOG.isTraceEnabled()) {
            LOG.trace("\n--- Block-2-Cell Instructions ---");
            for (Instruction i : b2cinst) {
                LOG.trace(i.toString());
            }
            LOG.trace("----------------------------------");
        }
    }
    // BEGIN FUNCTION PATCH
    if (!matricesNoReblock.isEmpty()) {
        for (int i = 0; i < matricesNoReblock.size(); i++) {
            String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
            String filename = scratchSpaceLoc + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR + _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Input";
            unBlockedFileNames.put(matricesNoReblock.get(i).getName(), filename);
        }
    }
    // null if no input matrices
    return b2cinst;
}

Example 22 with DataIdentifier

use of org.apache.sysml.parser.DataIdentifier in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method getOperands.

/**
 * Given a list of parameters as data identifiers, returns an array
 * of instruction operands.
 *
 * @param params list of data identifiers
 * @return operands
 */
protected CPOperand[] getOperands(ArrayList<DataIdentifier> params) {
    CPOperand[] ret = new CPOperand[params.size()];
    for (int i = 0; i < params.size(); i++) {
        DataIdentifier param = params.get(i);
        ret[i] = new CPOperand(param.getName(), param.getValueType(), param.getDataType());
    }
    return ret;
}

Example 23 with DataIdentifier

use of org.apache.sysml.parser.DataIdentifier in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method changeTmpInput.

private void changeTmpInput(long id, ExecutionContext ec) {
    ArrayList<DataIdentifier> inputParams = getInputParams();
    block2CellInst = getBlock2CellInstructions(inputParams, _unblockedFileNames);
    // post processing FUNCTION PATCH
    for (String var : _skipInReblock) {
        Data dat = ec.getVariable(var);
        if (dat instanceof MatrixObject)
            _unblockedFileNames.put(var, ((MatrixObject) dat).getFileName());
    }
}

Example 24 with DataIdentifier

use of org.apache.sysml.parser.DataIdentifier in project incubator-systemml by apache.

the class FunctionProgramBlock method checkOutputParameters.

protected void checkOutputParameters(LocalVariableMap vars) {
    for (DataIdentifier diOut : _outputParams) {
        String varName = diOut.getName();
        Data dat = vars.get(varName);
        if (dat == null)
            LOG.error("Function output " + varName + " is missing.");
        else if (dat.getDataType() != diOut.getDataType())
            LOG.warn("Function output " + varName + " has wrong data type: " + dat.getDataType() + ".");
        else if (dat.getValueType() != diOut.getValueType())
            LOG.warn("Function output " + varName + " has wrong value type: " + dat.getValueType() + ".");
    }
}

Example 25 with DataIdentifier

use of org.apache.sysml.parser.DataIdentifier in project incubator-systemml by apache.

the class ParForProgramBlock method createEmptyUnscopedVariables.

/**
 * Create empty matrix objects and scalars for all unscoped vars
 * (created within the parfor).
 *
 * NOTE: parfor gives no guarantees on the values of those objects - hence
 * we return -1 for sclars and empty matrix objects.
 *
 * @param out local variable map
 * @param sb statement block
 */
private static void createEmptyUnscopedVariables(LocalVariableMap out, StatementBlock sb) {
    VariableSet updated = sb.variablesUpdated();
    VariableSet livein = sb.liveIn();
    // for all vars IN <updated> AND NOT IN <livein>
    for (String var : updated.getVariableNames()) if (!livein.containsVariable(var)) {
        // create empty output
        DataIdentifier dat = updated.getVariable(var);
        DataType datatype = dat.getDataType();
        ValueType valuetype = dat.getValueType();
        Data dataObj = null;
        switch(datatype) {
            case SCALAR:
                switch(valuetype) {
                    case BOOLEAN:
                        dataObj = new BooleanObject(false);
                        break;
                    case INT:
                        dataObj = new IntObject(-1);
                        break;
                    case DOUBLE:
                        dataObj = new DoubleObject(-1d);
                        break;
                    case STRING:
                        dataObj = new StringObject("-1");
                        break;
                    default:
                        throw new DMLRuntimeException("Value type not supported: " + valuetype);
                }
                break;
            case MATRIX:
            case FRAME:
                // because metadata (e.g., outputinfo) not known at this place.
                break;
            case UNKNOWN:
                break;
            default:
                throw new DMLRuntimeException("Data type not supported: " + datatype);
        }
        if (dataObj != null)
            out.put(var, dataObj);
    }
}