Examples with DataType org.apache.sysml.parser.Expression.DataType used on opensource projects

Example 41 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class Recompiler method tryReadMetaDataFileMatrixCharacteristics.

private static void tryReadMetaDataFileMatrixCharacteristics(DataOp dop) {
    try {
        // get meta data filename
        String mtdname = DataExpression.getMTDFileName(dop.getFileName());
        Path path = new Path(mtdname);
        FileSystem fs = IOUtilFunctions.getFileSystem(mtdname);
        if (fs.exists(path)) {
            BufferedReader br = null;
            try {
                br = new BufferedReader(new InputStreamReader(fs.open(path)));
                JSONObject mtd = JSONHelper.parse(br);
                DataType dt = DataType.valueOf(String.valueOf(mtd.get(DataExpression.DATATYPEPARAM)).toUpperCase());
                dop.setDataType(dt);
                if (dt != DataType.FRAME)
                    dop.setValueType(ValueType.valueOf(String.valueOf(mtd.get(DataExpression.VALUETYPEPARAM)).toUpperCase()));
                dop.setDim1((dt == DataType.MATRIX || dt == DataType.FRAME) ? Long.parseLong(mtd.get(DataExpression.READROWPARAM).toString()) : 0);
                dop.setDim2((dt == DataType.MATRIX || dt == DataType.FRAME) ? Long.parseLong(mtd.get(DataExpression.READCOLPARAM).toString()) : 0);
            } finally {
                IOUtilFunctions.closeSilently(br);
            }
        }
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
}

Example 42 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class HopRewriteUtils method createAggUnaryOp.

public static AggUnaryOp createAggUnaryOp(Hop input, AggOp op, Direction dir) {
    DataType dt = (dir == Direction.RowCol) ? DataType.SCALAR : input.getDataType();
    AggUnaryOp auop = new AggUnaryOp(input.getName(), dt, input.getValueType(), op, dir, input);
    auop.setOutputBlocksizes(input.getRowsInBlock(), input.getColsInBlock());
    copyLineNumbers(input, auop);
    auop.refreshSizeInformation();
    return auop;
}

Example 43 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class ProgramConverter method parseDataObject.

/**
 * NOTE: MRJobConfiguration cannot be used for the general case because program blocks and
 * related symbol tables can be hierarchically structured.
 *
 * @param in data object as string
 * @return array of objects
 */
public static Object[] parseDataObject(String in) {
    Object[] ret = new Object[2];
    StringTokenizer st = new StringTokenizer(in, DATA_FIELD_DELIM);
    String name = st.nextToken();
    DataType datatype = DataType.valueOf(st.nextToken());
    ValueType valuetype = ValueType.valueOf(st.nextToken());
    String valString = st.hasMoreTokens() ? st.nextToken() : "";
    Data dat = null;
    switch(datatype) {
        case SCALAR:
            {
                switch(valuetype) {
                    case INT:
                        dat = new IntObject(Long.parseLong(valString));
                        break;
                    case DOUBLE:
                        dat = new DoubleObject(Double.parseDouble(valString));
                        break;
                    case BOOLEAN:
                        dat = new BooleanObject(Boolean.parseBoolean(valString));
                        break;
                    case STRING:
                        dat = new StringObject(valString);
                        break;
                    default:
                        throw new DMLRuntimeException("Unable to parse valuetype " + valuetype);
                }
                break;
            }
        case MATRIX:
            {
                MatrixObject mo = new MatrixObject(valuetype, valString);
                long rows = Long.parseLong(st.nextToken());
                long cols = Long.parseLong(st.nextToken());
                int brows = Integer.parseInt(st.nextToken());
                int bcols = Integer.parseInt(st.nextToken());
                long nnz = Long.parseLong(st.nextToken());
                InputInfo iin = InputInfo.stringToInputInfo(st.nextToken());
                OutputInfo oin = OutputInfo.stringToOutputInfo(st.nextToken());
                PartitionFormat partFormat = PartitionFormat.valueOf(st.nextToken());
                UpdateType inplace = UpdateType.valueOf(st.nextToken());
                MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, brows, bcols, nnz);
                MetaDataFormat md = new MetaDataFormat(mc, oin, iin);
                mo.setMetaData(md);
                if (partFormat._dpf != PDataPartitionFormat.NONE)
                    mo.setPartitioned(partFormat._dpf, partFormat._N);
                mo.setUpdateType(inplace);
                dat = mo;
                break;
            }
        default:
            throw new DMLRuntimeException("Unable to parse datatype " + datatype);
    }
    ret[0] = name;
    ret[1] = dat;
    return ret;
}

Example 44 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class ParForStatementBlock method validate.

@Override
public VariableSet validate(DMLProgram dmlProg, VariableSet ids, HashMap<String, ConstIdentifier> constVars, boolean conditional) {
    LOG.trace("PARFOR(" + _ID + "): validating ParForStatementBlock.");
    // create parent variable set via cloning
    _vsParent = new VariableSet(ids);
    if (// note: A is matrix, and A[i,1] is scalar
    LOG.isTraceEnabled())
        for (DataIdentifier di : _vsParent.getVariables().values()) LOG.trace("PARFOR: non-local " + di._name + ": " + di.getDataType().toString() + " with rowDim = " + di.getDim1());
    // normal validate via ForStatement (sequential)
    // NOTES:
    // * validate/dependency checking of nested parfor-loops happens at this point
    // * validate includes also constant propagation for from, to, incr expressions
    // * this includes also function inlining
    VariableSet vs = super.validate(dmlProg, ids, constVars, conditional);
    // check of correctness of specified parfor parameter names and
    // set default parameter values for all not specified parameters
    ParForStatement pfs = (ParForStatement) _statements.get(0);
    IterablePredicate predicate = pfs.getIterablePredicate();
    HashMap<String, String> params = predicate.getParForParams();
    if (// if parameter specified
    params != null) {
        // check for valid parameter types
        for (String key : params.keySet()) if (// always unconditional
        !_paramNames.contains(key))
            raiseValidateError("PARFOR: The specified parameter '" + key + "' is no valid parfor parameter.", false);
        // set defaults for all non-specified values
        // (except if CONSTRAINT optimizer, in order to distinguish specified parameters)
        boolean constrained = (params.containsKey(OPT_MODE) && params.get(OPT_MODE).equals(POptMode.CONSTRAINED.toString()));
        for (String key : _paramNames) if (!params.containsKey(key)) {
            if (constrained) {
                params.put(key, _paramDefaults2.get(key));
            } else // special treatment for degree of parallelism
            if (key.equals(PAR) && params.containsKey(EXEC_MODE) && params.get(EXEC_MODE).equals(PExecMode.REMOTE_MR.toString())) {
                int maxPMap = InfrastructureAnalyzer.getRemoteParallelMapTasks();
                // correction max number of reducers on yarn clusters
                if (InfrastructureAnalyzer.isYarnEnabled())
                    maxPMap = (int) Math.max(maxPMap, YarnClusterAnalyzer.getNumCores());
                params.put(key, String.valueOf(maxPMap));
            } else if (key.equals(PAR) && params.containsKey(EXEC_MODE) && params.get(EXEC_MODE).equals(PExecMode.REMOTE_MR_DP.toString())) {
                int maxPRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
                // correction max number of reducers on yarn clusters
                if (InfrastructureAnalyzer.isYarnEnabled())
                    maxPRed = (int) Math.max(maxPRed, YarnClusterAnalyzer.getNumCores() / 2);
                params.put(key, String.valueOf(maxPRed));
            } else
                // default case
                params.put(key, _paramDefaults.get(key));
        }
    } else {
        // set all defaults
        params = new HashMap<>();
        params.putAll(_paramDefaults);
        predicate.setParForParams(params);
    }
    // start time measurement for normalization and dependency analysis
    Timing time = new Timing(true);
    // LOOP DEPENDENCY ANALYSIS (test for dependency existence)
    // no false negative guaranteed, but possibly false positives
    /* Basic intuition: WRITES to NON-local variables are only permitted iff
		 *   - no data dep (no read other than own iteration w i < r j)
		 *   - no anti dep (no read other than own iteration w i > r j)
		 *   - no output dep (no write other than own iteration)
		 *   
		 * ALGORITHM:
		 * 1) Determine candidates C (writes to non-local variables)
		 * 2) Prune all c from C where no dependencies --> C'
		 * 3) Raise an exception/warning if C' not the empty set 
		 * 
		 * RESTRICTIONS:
		 * - array subscripts of non-local variables must be linear functions of the form 
		 *   a0+ a1*i + ... + a2*j, where i and j are for or parfor indexes.
		 * - for and parfor increments must be integer values 
		 * - only static (integer lower, upper bounds) range indexing
		 * - only input variables considered as potential candidates for checking 
		 * 
		 *   (TODO: in order to remove the last restriction, dependencies must be checked again after 
		 *   live variable analysis against LIVEOUT)
		 * 
		 * NOTE: validity is only checked during compilation, i.e., for dynamic from, to, incr MIN MAX values assumed.
		 */
    LOG.trace("PARFOR: running loop dependency analysis ...");
    // ### Step 1 ###: determine candidate set C
    HashSet<Candidate> C = new HashSet<>();
    HashSet<Candidate> C2 = new HashSet<>();
    // object for call by ref
    Integer sCount = 0;
    rDetermineCandidates(pfs.getBody(), C, sCount);
    if (LOG.isTraceEnabled())
        for (Candidate c : C) LOG.trace("PARFOR: dependency candidate: var '" + c._var + "' (accum=" + c._isAccum + ")");
    boolean check = (Integer.parseInt(params.get(CHECK)) == 1);
    if (check) {
        // ### Step 2 ###: prune c without dependencies
        _bounds = new Bounds();
        for (FunctionStatementBlock fsb : dmlProg.getFunctionStatementBlocks()) // writes to _bounds
        rDetermineBounds(fsb, false);
        // writes to _bounds
        rDetermineBounds(dmlProg.getStatementBlocks(), false);
        for (Candidate c : C) {
            // might be different in DataIdentifier
            DataType cdt = _vsParent.getVariables().get(c._var).getDataType();
            // assume no dependency
            sCount = 0;
            // output, data, anti
            boolean[] dep = new boolean[] { false, false, false };
            rCheckCandidates(c, cdt, pfs.getBody(), sCount, dep);
            if (LOG.isTraceEnabled()) {
                if (dep[0])
                    LOG.trace("PARFOR: output dependency detected for var '" + c._var + "'.");
                if (dep[1])
                    LOG.trace("PARFOR: data dependency detected for var '" + c._var + "'.");
                if (dep[2])
                    LOG.trace("PARFOR: anti dependency detected for var '" + c._var + "'.");
            }
            if (dep[0] || dep[1] || dep[2]) {
                C2.add(c);
                if (ABORT_ON_FIRST_DEPENDENCY)
                    break;
            }
        }
        // ### Step 3 ###: raise an exception / warning
        if (C2.size() > 0) {
            LOG.trace("PARFOR: loop dependencies detected.");
            StringBuilder depVars = new StringBuilder();
            for (Candidate c : C2) {
                if (depVars.length() > 0)
                    depVars.append(", ");
                depVars.append(c._var);
            }
            // always unconditional (to ensure we always raise dependency issues)
            raiseValidateError("PARFOR loop dependency analysis: " + "inter-iteration (loop-carried) dependencies detected for variable(s): " + depVars.toString() + ". \n " + "Please, ensure independence of iterations.", false);
        } else {
            LOG.trace("PARFOR: no loop dependencies detected.");
        }
    } else {
        LOG.debug("INFO: PARFOR(" + _ID + "): loop dependency analysis skipped.");
    }
    // a) add own candidates
    for (Candidate var : C) if (check || var._dat.getDataType() != DataType.SCALAR)
        addToResultVariablesNoDup(var._var, var._isAccum);
    // b) get and add child result vars (if required)
    ArrayList<ResultVar> tmp = new ArrayList<>();
    rConsolidateResultVars(pfs.getBody(), tmp);
    for (ResultVar var : tmp) if (_vsParent.containsVariable(var._name))
        addToResultVariablesNoDup(var);
    if (LDEBUG)
        for (ResultVar rvar : _resultVars) LOG.debug("INFO: PARFOR final result variable: " + rvar._name);
    // cleanup function cache in order to prevent side effects between parfor statements
    if (USE_FN_CACHE)
        _fncache.clear();
    LOG.debug("INFO: PARFOR(" + _ID + "): validate successful (no dependencies) in " + time.stop() + "ms.");
    return vs;
}

Example 45 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class ProgramConverter method parseDataIdentifier.

private static DataIdentifier parseDataIdentifier(String in) {
    StringTokenizer st = new StringTokenizer(in, DATA_FIELD_DELIM);
    String name = st.nextToken();
    DataType dt = DataType.valueOf(st.nextToken());
    ValueType vt = ValueType.valueOf(st.nextToken());
    DataIdentifier dat = new DataIdentifier(name);
    dat.setDataType(dt);
    dat.setValueType(vt);
    return dat;
}