Examples with IntObject org.apache.sysml.runtime.instructions.cp.IntObject used on opensource projects

Example 1 with IntObject

use of org.apache.sysml.runtime.instructions.cp.IntObject in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method verifyAndAttachOutputs.

/**
	 * Method to verify that function outputs match with declared outputs
	 * 
	 * @param ec execution context
	 * @param returnFunc package function
	 * @param outputParams output parameters
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
protected void verifyAndAttachOutputs(ExecutionContext ec, PackageFunction returnFunc, String outputParams) throws DMLRuntimeException {
    ArrayList<String> outputs = getParameters(outputParams);
    if (outputs.size() != returnFunc.getNumFunctionOutputs()) {
        throw new DMLRuntimeException("Number of function outputs (" + returnFunc.getNumFunctionOutputs() + ") " + "does not match with declaration (" + outputs.size() + ").");
    }
    // iterate over each output and verify that type matches
    for (int i = 0; i < outputs.size(); i++) {
        StringTokenizer tk = new StringTokenizer(outputs.get(i), ":");
        ArrayList<String> tokens = new ArrayList<String>();
        while (tk.hasMoreTokens()) {
            tokens.add(tk.nextToken());
        }
        if (returnFunc.getFunctionOutput(i).getType() == FunctionParameterType.Matrix) {
            Matrix m = (Matrix) returnFunc.getFunctionOutput(i);
            if (!(tokens.get(0).equals(getFunctionParameterDataTypeString(FunctionParameterType.Matrix))) || !(tokens.get(2).equals(getMatrixValueTypeString(m.getValueType())))) {
                throw new DMLRuntimeException("Function output '" + outputs.get(i) + "' does not match with declaration.");
            }
            // add result to variableMapping
            String varName = tokens.get(1);
            MatrixObject newVar = createOutputMatrixObject(m);
            newVar.setVarName(varName);
            //getVariables().put(varName, newVar); //put/override in local symbol table
            ec.setVariable(varName, newVar);
            continue;
        }
        if (returnFunc.getFunctionOutput(i).getType() == FunctionParameterType.Scalar) {
            Scalar s = (Scalar) returnFunc.getFunctionOutput(i);
            if (!tokens.get(0).equals(getFunctionParameterDataTypeString(FunctionParameterType.Scalar)) || !tokens.get(2).equals(getScalarValueTypeString(s.getScalarType()))) {
                throw new DMLRuntimeException("Function output '" + outputs.get(i) + "' does not match with declaration.");
            }
            // allocate and set appropriate object based on type
            ScalarObject scalarObject = null;
            ScalarValueType type = s.getScalarType();
            switch(type) {
                case Integer:
                    scalarObject = new IntObject(tokens.get(1), Long.parseLong(s.getValue()));
                    break;
                case Double:
                    scalarObject = new DoubleObject(tokens.get(1), Double.parseDouble(s.getValue()));
                    break;
                case Boolean:
                    scalarObject = new BooleanObject(tokens.get(1), Boolean.parseBoolean(s.getValue()));
                    break;
                case Text:
                    scalarObject = new StringObject(tokens.get(1), s.getValue());
                    break;
                default:
                    throw new DMLRuntimeException("Unknown scalar value type '" + type + "' of output '" + outputs.get(i) + "'.");
            }
            //this.getVariables().put(tokens.get(1), scalarObject);
            ec.setVariable(tokens.get(1), scalarObject);
            continue;
        }
        if (returnFunc.getFunctionOutput(i).getType() == FunctionParameterType.Object) {
            if (!tokens.get(0).equals(getFunctionParameterDataTypeString(FunctionParameterType.Object))) {
                throw new DMLRuntimeException("Function output '" + outputs.get(i) + "' does not match with declaration.");
            }
            throw new DMLRuntimeException("Object types not yet supported");
        // continue;
        }
        throw new DMLRuntimeException("Unknown data type '" + returnFunc.getFunctionOutput(i).getType() + "' " + "of output '" + outputs.get(i) + "'.");
    }
}

Example 2 with IntObject

use of org.apache.sysml.runtime.instructions.cp.IntObject in project incubator-systemml by apache.

the class RemoteDPParForSparkWorker method call.

@Override
public Iterator<Tuple2<Long, String>> call(Iterator<Tuple2<Long, Iterable<Writable>>> arg0) throws Exception {
    ArrayList<Tuple2<Long, String>> ret = new ArrayList<>();
    // lazy parworker initialization
    configureWorker(TaskContext.get().taskAttemptId());
    // process all matrix partitions of this data partition
    MatrixBlock partition = null;
    while (arg0.hasNext()) {
        Tuple2<Long, Iterable<Writable>> larg = arg0.next();
        // collect input partition (check via equals because oinfo deserialized instance)
        if (_oinfo.equals(OutputInfo.BinaryBlockOutputInfo))
            partition = collectBinaryBlock(larg._2(), partition);
        else
            partition = collectBinaryCellInput(larg._2());
        // update in-memory matrix partition
        MatrixObject mo = _ec.getMatrixObject(_inputVar);
        mo.setInMemoryPartition(partition);
        // create tasks for input data
        Task lTask = new Task(_iterVar, TaskType.SET);
        lTask.addIteration(new IntObject(larg._1()));
        // execute program
        long numIter = getExecutedIterations();
        super.executeTask(lTask);
        // maintain accumulators
        _aTasks.add(1);
        _aIters.add((int) (getExecutedIterations() - numIter));
    }
    // write output if required (matrix indexed write)
    ArrayList<String> tmp = RemoteParForUtils.exportResultVariables(_workerID, _ec.getVariables(), _resultVars);
    for (String val : tmp) ret.add(new Tuple2<>(_workerID, val));
    return ret.iterator();
}

Example 3 with IntObject

use of org.apache.sysml.runtime.instructions.cp.IntObject in project incubator-systemml by apache.

the class RemoteDPParWorkerReducer method reduce.

@Override
public void reduce(LongWritable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter) throws IOException {
    // cache collector/reporter (for write in close)
    _out = out;
    _report = reporter;
    // collect input partition
    if (_info == OutputInfo.BinaryBlockOutputInfo)
        _partition = collectBinaryBlock(valueList);
    else
        _partition = collectBinaryCellInput(valueList);
    // execute program
    LOG.trace("execute RemoteDPParWorkerReducer " + _stringID + " (" + _workerID + ")");
    try {
        // update in-memory matrix partition
        MatrixObject mo = _ec.getMatrixObject(_inputVar);
        mo.setInMemoryPartition(_partition);
        // create tasks for input data
        Task lTask = new Task(_iterVar, TaskType.SET);
        lTask.addIteration(new IntObject(key.get()));
        // execute program
        executeTask(lTask);
    } catch (Exception ex) {
        throw new IOException("ParFOR: Failed to execute task.", ex);
    }
    // statistic maintenance (after final export)
    RemoteParForUtils.incrementParForMRCounters(_report, 1, 1);
}

Example 4 with IntObject

use of org.apache.sysml.runtime.instructions.cp.IntObject in project incubator-systemml by apache.

the class RemoteParForColocatedFileSplit method getLocations.

/**
 * Get the list of hostnames where the input split is located.
 */
@Override
public String[] getLocations() throws IOException {
    // Timing time = new Timing();
    // time.start();
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    FileSystem fs = IOUtilFunctions.getFileSystem(getPath(), job);
    // read task string
    LongWritable key = new LongWritable();
    Text value = new Text();
    RecordReader<LongWritable, Text> reader = null;
    try {
        reader = new NLineInputFormat().getRecordReader(this, job, Reporter.NULL);
        reader.next(key, value);
    } finally {
        IOUtilFunctions.closeSilently(reader);
    }
    // parse task
    Task t = Task.parseCompactString(value.toString());
    // get all locations
    HashMap<String, Integer> hosts = new HashMap<>();
    if (t.getType() == TaskType.SET) {
        for (IntObject val : t.getIterations()) {
            String fname = _fname + "/" + String.valueOf(((val.getLongValue() - 1) / _blen + 1));
            FileStatus status = fs.getFileStatus(new Path(fname));
            BlockLocation[] tmp1 = fs.getFileBlockLocations(status, 0, status.getLen());
            for (BlockLocation bl : tmp1) countHosts(hosts, bl.getHosts());
        }
    } else // TaskType.RANGE
    {
        // since this is a serial process, we use just the first iteration
        // as a heuristic for location information
        long lFrom = t.getIterations().get(0).getLongValue();
        long lTo = t.getIterations().get(1).getLongValue();
        for (long li : new long[] { lFrom, lTo }) {
            String fname = _fname + "/" + String.valueOf(((li - 1) / _blen + 1));
            FileStatus status = fs.getFileStatus(new Path(fname));
            BlockLocation[] tmp1 = fs.getFileBlockLocations(status, 0, status.getLen());
            for (BlockLocation bl : tmp1) countHosts(hosts, bl.getHosts());
        }
    }
    // majority consensus on top host
    return getTopHosts(hosts);
}

Example 5 with IntObject

use of org.apache.sysml.runtime.instructions.cp.IntObject in project incubator-systemml by apache.

the class Task method toFormatedString.

public String toFormatedString() {
    StringBuilder sb = new StringBuilder();
    sb.append("task (type=");
    sb.append(_type);
    sb.append(", iterations={");
    int count = 0;
    for (IntObject dat : _iterations) {
        if (count != 0)
            sb.append(";");
        sb.append("[");
        sb.append(_iterVar);
        sb.append("=");
        sb.append(dat.getLongValue());
        sb.append("]");
        count++;
    }
    sb.append("})");
    return sb.toString();
}