Examples with DataIdentifier org.apache.sysml.parser.DataIdentifier used on opensource projects

Example 46 with DataIdentifier

use of org.apache.sysml.parser.DataIdentifier in project incubator-systemml by apache.

the class ProgramConverter method rParseExternalFunctionProgramBlock.

private static ExternalFunctionProgramBlock rParseExternalFunctionProgramBlock(String in, Program prog, int id) {
    String lin = in.substring(PARFOR_PB_EFC.length(), in.length() - PARFOR_PB_END.length());
    HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(lin, COMPONENTS_DELIM);
    // LocalVariableMap vars = parseVariables(st.nextToken());
    // inputs, outputs and params
    ArrayList<DataIdentifier> dat1 = parseDataIdentifiers(st.nextToken());
    ArrayList<DataIdentifier> dat2 = parseDataIdentifiers(st.nextToken());
    HashMap<String, String> dat3 = parseStringHashMap(st.nextToken());
    // basedir
    String basedir = st.nextToken();
    // instructions (required for removing INST BEGIN, END)
    parseInstructions(st.nextToken(), id);
    // program blocks
    ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id);
    ArrayList<DataIdentifier> tmp1 = new ArrayList<>(dat1);
    ArrayList<DataIdentifier> tmp2 = new ArrayList<>(dat2);
    // only CP external functions, because no nested MR jobs for reblocks
    ExternalFunctionProgramBlockCP efpb = new ExternalFunctionProgramBlockCP(prog, tmp1, tmp2, dat3, basedir);
    efpb.setChildBlocks(pbs);
    return efpb;
}

Example 47 with DataIdentifier

use of org.apache.sysml.parser.DataIdentifier in project incubator-systemml by apache.

the class ProgramConverter method parseDataIdentifier.

private static DataIdentifier parseDataIdentifier(String in) {
    StringTokenizer st = new StringTokenizer(in, DATA_FIELD_DELIM);
    String name = st.nextToken();
    DataType dt = DataType.valueOf(st.nextToken());
    ValueType vt = ValueType.valueOf(st.nextToken());
    DataIdentifier dat = new DataIdentifier(name);
    dat.setDataType(dt);
    dat.setValueType(vt);
    return dat;
}

Example 48 with DataIdentifier

use of org.apache.sysml.parser.DataIdentifier in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method execute.

/**
 * Method to be invoked to execute instructions for the external function
 * invocation
 */
@Override
public void execute(ExecutionContext ec) {
    _runID = _idSeq.getNextID();
    changeTmpInput(_runID, ec);
    changeTmpOutput(_runID);
    // export input variables to HDFS (see RunMRJobs)
    ArrayList<DataIdentifier> inputParams = null;
    try {
        inputParams = getInputParams();
        for (DataIdentifier di : inputParams) {
            Data d = ec.getVariable(di.getName());
            if (d.getDataType().isMatrix())
                ((MatrixObject) d).exportData();
        }
    } catch (Exception e) {
        throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error exporting input variables to HDFS", e);
    }
    // convert block to cell
    if (block2CellInst != null) {
        ArrayList<Instruction> tempInst = new ArrayList<>();
        tempInst.addAll(block2CellInst);
        try {
            this.executeInstructions(tempInst, ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error executing " + tempInst.toString(), e);
        }
    }
    // now execute package function
    for (int i = 0; i < _inst.size(); i++) {
        try {
            if (_inst.get(i) instanceof ExternalFunctionInvocationInstruction)
                ((ExternalFunctionInvocationInstruction) _inst.get(i)).processInstruction(ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction " + _inst.get(i).toString(), e);
        }
    }
    // convert cell to block
    if (cell2BlockInst != null) {
        ArrayList<Instruction> tempInst = new ArrayList<>();
        try {
            tempInst.clear();
            tempInst.addAll(cell2BlockInst);
            this.executeInstructions(tempInst, ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction " + cell2BlockInst.toString(), e);
        }
    }
    // check return values
    checkOutputParameters(ec.getVariables());
}

Example 49 with DataIdentifier

use of org.apache.sysml.parser.DataIdentifier in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method getCell2BlockInstructions.

/**
 * Method to generate a reblock job to convert the cell representation into block representation
 *
 * @param outputParams list out output data identifiers
 * @param blockedFileNames map of blocked file names
 * @return list of instructions
 */
private ArrayList<Instruction> getCell2BlockInstructions(ArrayList<DataIdentifier> outputParams, HashMap<String, String> blockedFileNames) {
    ArrayList<Instruction> c2binst = null;
    // list of matrices that need to be reblocked
    ArrayList<DataIdentifier> matrices = new ArrayList<>();
    ArrayList<DataIdentifier> matricesNoReblock = new ArrayList<>();
    // identify outputs that are matrices
    for (int i = 0; i < outputParams.size(); i++) {
        if (outputParams.get(i).getDataType().isMatrix()) {
            if (_skipOutReblock.contains(outputParams.get(i).getName()))
                matricesNoReblock.add(outputParams.get(i));
            else
                matrices.add(outputParams.get(i));
        }
    }
    if (!matrices.isEmpty()) {
        c2binst = new ArrayList<>();
        MRJobInstruction reblkInst = new MRJobInstruction(JobType.REBLOCK);
        TreeMap<Integer, ArrayList<String>> MRJobLineNumbers = null;
        if (DMLScript.ENABLE_DEBUG_MODE) {
            MRJobLineNumbers = new TreeMap<>();
        }
        ArrayList<String> inLabels = new ArrayList<>();
        ArrayList<String> outLabels = new ArrayList<>();
        String[] outputs = new String[matrices.size()];
        byte[] resultIndex = new byte[matrices.size()];
        String reblock = "";
        // Keep a copy of a single MR reblock instruction
        String reblockStr = "";
        String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
        try {
            // create a RBLK job that transforms each output matrix from cell to block
            for (int i = 0; i < matrices.size(); i++) {
                inLabels.add(matrices.get(i).getName());
                outLabels.add(matrices.get(i).getName() + "_extFnOutput");
                outputs[i] = scratchSpaceLoc + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR + _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Output";
                blockedFileNames.put(matrices.get(i).getName(), outputs[i]);
                // (matrices.size()+i);
                resultIndex[i] = (byte) i;
                if (i > 0)
                    reblock += Lop.INSTRUCTION_DELIMITOR;
                reblock += "MR" + Lop.OPERAND_DELIMITOR + "rblk" + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + "true";
                if (DMLScript.ENABLE_DEBUG_MODE) {
                    // Create a copy of reblock instruction but as a single instruction (FOR DEBUGGER)
                    reblockStr = "MR" + Lop.OPERAND_DELIMITOR + "rblk" + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + "true";
                    // Set MR reblock instruction line number (FOR DEBUGGER)
                    if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
                        MRJobLineNumbers.put(matrices.get(i).getBeginLine(), new ArrayList<String>());
                    }
                    MRJobLineNumbers.get(matrices.get(i).getBeginLine()).add(reblockStr);
                }
                // create metadata instructions to populate symbol table
                // with variables that hold blocked matrices
                Instruction createInst = VariableCPInstruction.prepareCreateMatrixVariableInstruction(outLabels.get(i), outputs[i], false, OutputInfo.outputInfoToString(OutputInfo.BinaryBlockOutputInfo));
                createInst.setLocation(matrices.get(i));
                c2binst.add(createInst);
            }
            reblkInst.setReBlockInstructions(inLabels.toArray(new String[inLabels.size()]), "", reblock, "", outLabels.toArray(new String[inLabels.size()]), resultIndex, 1, 1);
            c2binst.add(reblkInst);
            // generate instructions that rename the output variables of REBLOCK job
            Instruction cpInst = null, rmInst = null;
            for (int i = 0; i < matrices.size(); i++) {
                cpInst = VariableCPInstruction.prepareCopyInstruction(outLabels.get(i), matrices.get(i).getName());
                rmInst = VariableCPInstruction.prepareRemoveInstruction(outLabels.get(i));
                cpInst.setLocation(matrices.get(i));
                rmInst.setLocation(matrices.get(i));
                c2binst.add(cpInst);
                c2binst.add(rmInst);
            // c2binst.add(CPInstructionParser.parseSingleInstruction("CP" + Lops.OPERAND_DELIMITOR + "cpvar"+Lops.OPERAND_DELIMITOR+ outLabels.get(i) + Lops.OPERAND_DELIMITOR + matrices.get(i).getName()));
            }
        } catch (Exception e) {
            throw new RuntimeException(this.printBlockErrorLocation() + "error generating instructions", e);
        }
        // LOGGING instructions
        if (LOG.isTraceEnabled()) {
            LOG.trace("\n--- Cell-2-Block Instructions ---");
            for (Instruction i : c2binst) {
                LOG.trace(i.toString());
            }
            LOG.trace("----------------------------------");
        }
    }
    // null if no output matrices
    return c2binst;
}

Example 50 with DataIdentifier

use of org.apache.sysml.parser.DataIdentifier in project incubator-systemml by apache.

the class PydmlSyntacticValidator method exitIfdefAssignmentStatement.

@Override
public void exitIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) {
    if (!ctx.commandLineParam.getText().startsWith("$")) {
        notifyErrorListeners("the first argument of ifdef function should be a commandline argument parameter (which starts with $)", ctx.commandLineParam.start);
        return;
    }
    if (ctx.targetList == null) {
        notifyErrorListeners("incorrect lvalue in ifdef function ", ctx.start);
        return;
    }
    String targetListText = ctx.targetList.getText();
    if (targetListText.startsWith("$")) {
        notifyErrorListeners("lhs of ifdef function cannot be a commandline parameters. Use local variable instead", ctx.start);
        return;
    }
    DataIdentifier target = null;
    if (ctx.targetList.dataInfo.expr instanceof DataIdentifier) {
        target = (DataIdentifier) ctx.targetList.dataInfo.expr;
        Expression source = null;
        if (ctx.commandLineParam.dataInfo.expr != null) {
            // Since commandline parameter is set
            // The check of following is done in fillExpressionInfoCommandLineParameters:
            // Command line param cannot be empty string
            // If you want to pass space, please quote it
            source = ctx.commandLineParam.dataInfo.expr;
        } else {
            source = ctx.source.info.expr;
        }
        try {
            ctx.info.stmt = new AssignmentStatement(ctx, target, source, currentFile);
        } catch (LanguageException e) {
            notifyErrorListeners("invalid assignment for ifdef function", ctx.targetList.start);
            return;
        }
    } else {
        notifyErrorListeners("incorrect lvalue in ifdef function ", ctx.targetList.start);
        return;
    }
}