Examples with DataType org.apache.sysml.parser.Expression.DataType used on opensource projects

Example 16 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class FrameMatrixCastingTest method runFrameCastingTest.

/**
 * @param testname
 * @param schema
 * @param wildcard
 */
private void runFrameCastingTest(String testname, boolean multColBlks, ValueType vt, ExecType et) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        int cols = multColBlks ? cols2 : cols1;
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), output("B") };
        // data generation
        double[][] A = getRandomMatrix(rows, cols, -1, 1, 0.9, 7);
        DataType dtin = testname.equals(TEST_NAME1) ? DataType.FRAME : DataType.MATRIX;
        ValueType vtin = testname.equals(TEST_NAME1) ? vt : ValueType.DOUBLE;
        writeMatrixOrFrameInput(input("A"), A, rows, cols, dtin, vtin);
        // run testcase
        runTest(true, false, null, -1);
        // compare matrices
        DataType dtout = testname.equals(TEST_NAME1) ? DataType.MATRIX : DataType.FRAME;
        double[][] B = readMatrixOrFrameInput(output("B"), rows, cols, dtout);
        TestUtils.compareMatrices(A, B, rows, cols, 0);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 17 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class ParForStatementBlock method validate.

@Override
public VariableSet validate(DMLProgram dmlProg, VariableSet ids, HashMap<String, ConstIdentifier> constVars, boolean conditional) {
    LOG.trace("PARFOR(" + _ID + "): validating ParForStatementBlock.");
    // create parent variable set via cloning
    _vsParent = new VariableSet(ids);
    if (// note: A is matrix, and A[i,1] is scalar
    LOG.isTraceEnabled())
        for (DataIdentifier di : _vsParent.getVariables().values()) LOG.trace("PARFOR: non-local " + di._name + ": " + di.getDataType().toString() + " with rowDim = " + di.getDim1());
    // normal validate via ForStatement (sequential)
    // NOTES:
    // * validate/dependency checking of nested parfor-loops happens at this point
    // * validate includes also constant propagation for from, to, incr expressions
    // * this includes also function inlining
    VariableSet vs = super.validate(dmlProg, ids, constVars, conditional);
    // check of correctness of specified parfor parameter names and
    // set default parameter values for all not specified parameters
    ParForStatement pfs = (ParForStatement) _statements.get(0);
    IterablePredicate predicate = pfs.getIterablePredicate();
    HashMap<String, String> params = predicate.getParForParams();
    if (// if parameter specified
    params != null) {
        // check for valid parameter types
        for (String key : params.keySet()) if (// always unconditional
        !_paramNames.contains(key))
            raiseValidateError("PARFOR: The specified parameter '" + key + "' is no valid parfor parameter.", false);
        // set defaults for all non-specified values
        // (except if CONSTRAINT optimizer, in order to distinguish specified parameters)
        boolean constrained = (params.containsKey(OPT_MODE) && params.get(OPT_MODE).equals(POptMode.CONSTRAINED.toString()));
        for (String key : _paramNames) if (!params.containsKey(key)) {
            if (constrained) {
                params.put(key, _paramDefaults2.get(key));
            } else // special treatment for degree of parallelism
            if (key.equals(PAR) && params.containsKey(EXEC_MODE) && params.get(EXEC_MODE).equals(PExecMode.REMOTE_MR.toString())) {
                int maxPMap = InfrastructureAnalyzer.getRemoteParallelMapTasks();
                // correction max number of reducers on yarn clusters
                if (InfrastructureAnalyzer.isYarnEnabled())
                    maxPMap = (int) Math.max(maxPMap, YarnClusterAnalyzer.getNumCores());
                params.put(key, String.valueOf(maxPMap));
            } else if (key.equals(PAR) && params.containsKey(EXEC_MODE) && params.get(EXEC_MODE).equals(PExecMode.REMOTE_MR_DP.toString())) {
                int maxPRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
                // correction max number of reducers on yarn clusters
                if (InfrastructureAnalyzer.isYarnEnabled())
                    maxPRed = (int) Math.max(maxPRed, YarnClusterAnalyzer.getNumCores() / 2);
                params.put(key, String.valueOf(maxPRed));
            } else
                // default case
                params.put(key, _paramDefaults.get(key));
        }
    } else {
        // set all defaults
        params = new HashMap<>();
        params.putAll(_paramDefaults);
        predicate.setParForParams(params);
    }
    // start time measurement for normalization and dependency analysis
    Timing time = new Timing(true);
    // LOOP DEPENDENCY ANALYSIS (test for dependency existence)
    // no false negative guaranteed, but possibly false positives
    /* Basic intuition: WRITES to NON-local variables are only permitted iff
		 *   - no data dep (no read other than own iteration w i < r j)
		 *   - no anti dep (no read other than own iteration w i > r j)
		 *   - no output dep (no write other than own iteration)
		 *   
		 * ALGORITHM:
		 * 1) Determine candidates C (writes to non-local variables)
		 * 2) Prune all c from C where no dependencies --> C'
		 * 3) Raise an exception/warning if C' not the empty set 
		 * 
		 * RESTRICTIONS:
		 * - array subscripts of non-local variables must be linear functions of the form 
		 *   a0+ a1*i + ... + a2*j, where i and j are for or parfor indexes.
		 * - for and parfor increments must be integer values 
		 * - only static (integer lower, upper bounds) range indexing
		 * - only input variables considered as potential candidates for checking 
		 * 
		 *   (TODO: in order to remove the last restriction, dependencies must be checked again after 
		 *   live variable analysis against LIVEOUT)
		 * 
		 * NOTE: validity is only checked during compilation, i.e., for dynamic from, to, incr MIN MAX values assumed.
		 */
    LOG.trace("PARFOR: running loop dependency analysis ...");
    // ### Step 1 ###: determine candidate set C
    HashSet<Candidate> C = new HashSet<>();
    HashSet<Candidate> C2 = new HashSet<>();
    // object for call by ref
    Integer sCount = 0;
    rDetermineCandidates(pfs.getBody(), C, sCount);
    if (LOG.isTraceEnabled())
        for (Candidate c : C) LOG.trace("PARFOR: dependency candidate: var '" + c._var + "' (accum=" + c._isAccum + ")");
    boolean check = (Integer.parseInt(params.get(CHECK)) == 1);
    if (check) {
        // ### Step 2 ###: prune c without dependencies
        _bounds = new Bounds();
        for (FunctionStatementBlock fsb : dmlProg.getFunctionStatementBlocks()) // writes to _bounds
        rDetermineBounds(fsb, false);
        // writes to _bounds
        rDetermineBounds(dmlProg.getStatementBlocks(), false);
        for (Candidate c : C) {
            // might be different in DataIdentifier
            DataType cdt = _vsParent.getVariables().get(c._var).getDataType();
            // assume no dependency
            sCount = 0;
            // output, data, anti
            boolean[] dep = new boolean[] { false, false, false };
            rCheckCandidates(c, cdt, pfs.getBody(), sCount, dep);
            if (LOG.isTraceEnabled()) {
                if (dep[0])
                    LOG.trace("PARFOR: output dependency detected for var '" + c._var + "'.");
                if (dep[1])
                    LOG.trace("PARFOR: data dependency detected for var '" + c._var + "'.");
                if (dep[2])
                    LOG.trace("PARFOR: anti dependency detected for var '" + c._var + "'.");
            }
            if (dep[0] || dep[1] || dep[2]) {
                C2.add(c);
                if (ABORT_ON_FIRST_DEPENDENCY)
                    break;
            }
        }
        // ### Step 3 ###: raise an exception / warning
        if (C2.size() > 0) {
            LOG.trace("PARFOR: loop dependencies detected.");
            StringBuilder depVars = new StringBuilder();
            for (Candidate c : C2) {
                if (depVars.length() > 0)
                    depVars.append(", ");
                depVars.append(c._var);
            }
            // always unconditional (to ensure we always raise dependency issues)
            raiseValidateError("PARFOR loop dependency analysis: " + "inter-iteration (loop-carried) dependencies detected for variable(s): " + depVars.toString() + ". \n " + "Please, ensure independence of iterations.", false);
        } else {
            LOG.trace("PARFOR: no loop dependencies detected.");
        }
    } else {
        LOG.debug("INFO: PARFOR(" + _ID + "): loop dependency analysis skipped.");
    }
    // a) add own candidates
    for (Candidate var : C) if (check || var._dat.getDataType() != DataType.SCALAR)
        addToResultVariablesNoDup(var._var, var._isAccum);
    // b) get and add child result vars (if required)
    ArrayList<ResultVar> tmp = new ArrayList<>();
    rConsolidateResultVars(pfs.getBody(), tmp);
    for (ResultVar var : tmp) if (_vsParent.containsVariable(var._name))
        addToResultVariablesNoDup(var);
    if (LDEBUG)
        for (ResultVar rvar : _resultVars) LOG.debug("INFO: PARFOR final result variable: " + rvar._name);
    // cleanup function cache in order to prevent side effects between parfor statements
    if (USE_FN_CACHE)
        _fncache.clear();
    LOG.debug("INFO: PARFOR(" + _ID + "): validate successful (no dependencies) in " + time.stop() + "ms.");
    return vs;
}

Example 18 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class VariableCPInstruction method parseInstruction.

public static VariableCPInstruction parseInstruction(String str) {
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    String opcode = parts[0];
    VariableOperationCode voc = getVariableOperationCode(opcode);
    if (voc == VariableOperationCode.CreateVariable) {
        if (// && parts.length != 10 )
        parts.length < 5)
            throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
    } else if (voc == VariableOperationCode.MoveVariable) {
        // mvvar tempA A; or mvvar mvar5 "data/out.mtx" "binary"
        if (parts.length != 3 && parts.length != 4)
            throw new DMLRuntimeException("Invalid number of operands in mvvar instruction: " + str);
    } else if (voc == VariableOperationCode.Write) {
        // Write instructions for csv files also include three additional parameters (hasHeader, delimiter, sparse)
        if (parts.length != 5 && parts.length != 8)
            throw new DMLRuntimeException("Invalid number of operands in write instruction: " + str);
    } else {
        if (voc != VariableOperationCode.RemoveVariable)
            // no output
            InstructionUtils.checkNumFields(parts, getArity(voc));
    }
    CPOperand in1 = null, in2 = null, in3 = null, in4 = null, out = null;
    switch(voc) {
        case CreateVariable:
            // variable name
            DataType dt = DataType.valueOf(parts[4]);
            ValueType vt = dt == DataType.MATRIX ? ValueType.DOUBLE : ValueType.STRING;
            int extSchema = (dt == DataType.FRAME && parts.length >= 13) ? 1 : 0;
            in1 = new CPOperand(parts[1], vt, dt);
            // file name
            in2 = new CPOperand(parts[2], ValueType.STRING, DataType.SCALAR);
            // file name override flag (always literal)
            in3 = new CPOperand(parts[3], ValueType.BOOLEAN, DataType.SCALAR);
            // format
            String fmt = parts[5];
            if (fmt.equalsIgnoreCase("csv")) {
                // 14 inputs: createvar corresponding to READ -- includes properties hasHeader, delim, fill, and fillValue
                if (parts.length < 15 + extSchema || parts.length > 17 + extSchema)
                    throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            } else {
                if (parts.length != 6 && parts.length != 12 + extSchema)
                    throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            }
            OutputInfo oi = OutputInfo.stringToOutputInfo(fmt);
            InputInfo ii = OutputInfo.getMatchingInputInfo(oi);
            MatrixCharacteristics mc = new MatrixCharacteristics();
            if (parts.length == 6) {
            // do nothing
            } else if (parts.length >= 11) {
                // matrix characteristics
                mc.setDimension(Long.parseLong(parts[6]), Long.parseLong(parts[7]));
                mc.setBlockSize(Integer.parseInt(parts[8]), Integer.parseInt(parts[9]));
                mc.setNonZeros(Long.parseLong(parts[10]));
            } else {
                throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            }
            MetaDataFormat iimd = new MetaDataFormat(mc, oi, ii);
            UpdateType updateType = UpdateType.COPY;
            if (parts.length >= 12)
                updateType = UpdateType.valueOf(parts[11].toUpperCase());
            // handle frame schema
            String schema = (dt == DataType.FRAME && parts.length >= 13) ? parts[parts.length - 1] : null;
            if (fmt.equalsIgnoreCase("csv")) {
                // Cretevar instructions for CSV format either has 13 or 14 inputs.
                // 13 inputs: createvar corresponding to WRITE -- includes properties hasHeader, delim, and sparse
                // 14 inputs: createvar corresponding to READ -- includes properties hasHeader, delim, fill, and fillValue
                FileFormatProperties fmtProperties = null;
                if (parts.length == 15 + extSchema) {
                    boolean hasHeader = Boolean.parseBoolean(parts[12]);
                    String delim = parts[13];
                    boolean sparse = Boolean.parseBoolean(parts[14]);
                    fmtProperties = new CSVFileFormatProperties(hasHeader, delim, sparse);
                } else {
                    boolean hasHeader = Boolean.parseBoolean(parts[12]);
                    String delim = parts[13];
                    boolean fill = Boolean.parseBoolean(parts[14]);
                    double fillValue = UtilFunctions.parseToDouble(parts[15]);
                    String naStrings = null;
                    if (parts.length == 17 + extSchema)
                        naStrings = parts[16];
                    fmtProperties = new CSVFileFormatProperties(hasHeader, delim, fill, fillValue, naStrings);
                }
                return new VariableCPInstruction(VariableOperationCode.CreateVariable, in1, in2, in3, iimd, updateType, fmtProperties, schema, opcode, str);
            } else {
                return new VariableCPInstruction(VariableOperationCode.CreateVariable, in1, in2, in3, iimd, updateType, schema, opcode, str);
            }
        case AssignVariable:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            break;
        case CopyVariable:
            // Value types are not given here
            in1 = new CPOperand(parts[1], ValueType.UNKNOWN, DataType.UNKNOWN);
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            break;
        case MoveVariable:
            in1 = new CPOperand(parts[1], ValueType.UNKNOWN, DataType.UNKNOWN);
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            if (parts.length > 3)
                in3 = new CPOperand(parts[3], ValueType.UNKNOWN, DataType.UNKNOWN);
            break;
        case RemoveVariable:
            VariableCPInstruction rminst = new VariableCPInstruction(getVariableOperationCode(opcode), null, null, null, out, opcode, str);
            for (int i = 1; i < parts.length; i++) rminst.addInput(new CPOperand(parts[i], ValueType.UNKNOWN, DataType.SCALAR));
            return rminst;
        case RemoveVariableAndFile:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            // second argument must be a boolean
            if (in2.getValueType() != ValueType.BOOLEAN)
                throw new DMLRuntimeException("Unexpected value type for second argument in: " + str);
            break;
        case CastAsScalarVariable:
        case CastAsMatrixVariable:
        case CastAsFrameVariable:
        case CastAsDoubleVariable:
        case CastAsIntegerVariable:
        case CastAsBooleanVariable:
            // first operand is a variable name => string value type
            in1 = new CPOperand(parts[1]);
            // output variable name
            out = new CPOperand(parts[2]);
            break;
        case Write:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            in3 = new CPOperand(parts[3]);
            FileFormatProperties fprops = null;
            if (in3.getName().equalsIgnoreCase("csv")) {
                boolean hasHeader = Boolean.parseBoolean(parts[4]);
                String delim = parts[5];
                boolean sparse = Boolean.parseBoolean(parts[6]);
                fprops = new CSVFileFormatProperties(hasHeader, delim, sparse);
                // description
                in4 = new CPOperand(parts[7]);
            } else {
                fprops = new FileFormatProperties();
                // description
                in4 = new CPOperand(parts[4]);
            }
            VariableCPInstruction inst = new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, null, fprops, null, null, opcode, str);
            inst.addInput(in4);
            return inst;
        case Read:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            out = null;
            break;
        case SetFileName:
            // variable name
            in1 = new CPOperand(parts[1]);
            // file name
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            // option: remote or local
            in3 = new CPOperand(parts[3], ValueType.UNKNOWN, DataType.UNKNOWN);
            // return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, str);
            break;
    }
    return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, opcode, str);
}

Example 19 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class ArithmeticBinaryGPUInstruction method parseInstruction.

public static ArithmeticBinaryGPUInstruction parseInstruction(String str) {
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    InstructionUtils.checkNumFields(parts, 3);
    String opcode = parts[0];
    CPOperand in1 = new CPOperand(parts[1]);
    CPOperand in2 = new CPOperand(parts[2]);
    CPOperand out = new CPOperand(parts[3]);
    DataType dt1 = in1.getDataType();
    DataType dt2 = in2.getDataType();
    DataType dt3 = out.getDataType();
    Operator operator = (dt1 != dt2) ? InstructionUtils.parseScalarBinaryOperator(opcode, (dt1 == DataType.SCALAR)) : InstructionUtils.parseBinaryOperator(opcode);
    if (dt1 == DataType.MATRIX && dt2 == DataType.MATRIX && dt3 == DataType.MATRIX) {
        return new MatrixMatrixArithmeticGPUInstruction(operator, in1, in2, out, opcode, str);
    } else if (dt3 == DataType.MATRIX && ((dt1 == DataType.SCALAR && dt2 == DataType.MATRIX) || (dt1 == DataType.MATRIX && dt2 == DataType.SCALAR))) {
        return new ScalarMatrixArithmeticGPUInstruction(operator, in1, in2, out, opcode, str);
    } else
        throw new DMLRuntimeException("Unsupported GPU ArithmeticInstruction.");
}

Example 20 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class MatrixMatrixAxpyGPUInstruction method parseInstruction.

public static MatrixMatrixAxpyGPUInstruction parseInstruction(String str) {
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    InstructionUtils.checkNumFields(parts, 4);
    String opcode = parts[0];
    int multiplier = 1;
    if (opcode.equals("-*"))
        multiplier = -1;
    CPOperand in1 = new CPOperand(parts[1]);
    CPOperand constant = new CPOperand(parts[2]);
    if (constant.getDataType() != DataType.SCALAR)
        throw new DMLRuntimeException("Expected second operand to be a scalar");
    CPOperand in2 = new CPOperand(parts[3]);
    CPOperand out = new CPOperand(parts[4]);
    DataType dt1 = in1.getDataType();
    DataType dt2 = in2.getDataType();
    DataType dt3 = out.getDataType();
    Operator operator = (dt1 != dt2) ? InstructionUtils.parseScalarBinaryOperator(opcode, (dt1 == DataType.SCALAR)) : InstructionUtils.parseTernaryOperator(opcode);
    if (dt1 == DataType.MATRIX && dt2 == DataType.MATRIX && dt3 == DataType.MATRIX) {
        return new MatrixMatrixAxpyGPUInstruction(operator, in1, constant, multiplier, in2, out, opcode, str);
    } else if (dt3 == DataType.MATRIX && ((dt1 == DataType.SCALAR && dt2 == DataType.MATRIX) || (dt1 == DataType.MATRIX && dt2 == DataType.SCALAR))) {
        throw new DMLRuntimeException("Unsupported GPU PlusMult/MinusMult ArithmeticInstruction.");
    // return new ScalarMatrixArithmeticGPUInstruction(operator, in1, in2, out, opcode, str);
    } else
        throw new DMLRuntimeException("Unsupported GPU ArithmeticInstruction.");
}