Examples with IntObject org.apache.sysml.runtime.instructions.cp.IntObject used on opensource projects

Example 16 with IntObject

use of org.apache.sysml.runtime.instructions.cp.IntObject in project systemml by apache.

the class RemoteDPParWorkerReducer method reduce.

@Override
public void reduce(LongWritable key, Iterator<Writable> valueList, OutputCollector<Writable, Writable> out, Reporter reporter) throws IOException {
    // cache collector/reporter (for write in close)
    _out = out;
    _report = reporter;
    // collect input partition
    if (_info == OutputInfo.BinaryBlockOutputInfo)
        _partition = collectBinaryBlock(valueList);
    else
        _partition = collectBinaryCellInput(valueList);
    // execute program
    LOG.trace("execute RemoteDPParWorkerReducer " + _stringID + " (" + _workerID + ")");
    try {
        // update in-memory matrix partition
        MatrixObject mo = _ec.getMatrixObject(_inputVar);
        mo.setInMemoryPartition(_partition);
        // create tasks for input data
        Task lTask = new Task(_iterVar, TaskType.SET);
        lTask.addIteration(new IntObject(key.get()));
        // execute program
        executeTask(lTask);
    } catch (Exception ex) {
        throw new IOException("ParFOR: Failed to execute task.", ex);
    }
    // statistic maintenance (after final export)
    RemoteParForUtils.incrementParForMRCounters(_report, 1, 1);
}

Example 17 with IntObject

use of org.apache.sysml.runtime.instructions.cp.IntObject in project systemml by apache.

the class RemoteParForColocatedFileSplit method getLocations.

/**
 * Get the list of hostnames where the input split is located.
 */
@Override
public String[] getLocations() throws IOException {
    // Timing time = new Timing();
    // time.start();
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    FileSystem fs = IOUtilFunctions.getFileSystem(getPath(), job);
    // read task string
    LongWritable key = new LongWritable();
    Text value = new Text();
    RecordReader<LongWritable, Text> reader = null;
    try {
        reader = new NLineInputFormat().getRecordReader(this, job, Reporter.NULL);
        reader.next(key, value);
    } finally {
        IOUtilFunctions.closeSilently(reader);
    }
    // parse task
    Task t = Task.parseCompactString(value.toString());
    // get all locations
    HashMap<String, Integer> hosts = new HashMap<>();
    if (t.getType() == TaskType.SET) {
        for (IntObject val : t.getIterations()) {
            String fname = _fname + "/" + String.valueOf(((val.getLongValue() - 1) / _blen + 1));
            FileStatus status = fs.getFileStatus(new Path(fname));
            BlockLocation[] tmp1 = fs.getFileBlockLocations(status, 0, status.getLen());
            for (BlockLocation bl : tmp1) countHosts(hosts, bl.getHosts());
        }
    } else // TaskType.RANGE
    {
        // since this is a serial process, we use just the first iteration
        // as a heuristic for location information
        long lFrom = t.getIterations().get(0).getLongValue();
        long lTo = t.getIterations().get(1).getLongValue();
        for (long li : new long[] { lFrom, lTo }) {
            String fname = _fname + "/" + String.valueOf(((li - 1) / _blen + 1));
            FileStatus status = fs.getFileStatus(new Path(fname));
            BlockLocation[] tmp1 = fs.getFileBlockLocations(status, 0, status.getLen());
            for (BlockLocation bl : tmp1) countHosts(hosts, bl.getHosts());
        }
    }
    // majority consensus on top host
    return getTopHosts(hosts);
}

Example 18 with IntObject

use of org.apache.sysml.runtime.instructions.cp.IntObject in project systemml by apache.

the class Task method parseCompactString.

public static Task parseCompactString(String stask) {
    StringTokenizer st = new StringTokenizer(stask.trim(), ".");
    TaskType type = TaskType.valueOf(st.nextToken());
    String meta = st.nextToken();
    Task newTask = new Task(meta, type);
    // iteration data
    String sdata = st.nextToken();
    // remove brackets
    sdata = sdata.substring(1, sdata.length() - 1);
    StringTokenizer st2 = new StringTokenizer(sdata, ",");
    while (st2.hasMoreTokens()) {
        // create new iteration
        String lsdata = st2.nextToken();
        IntObject ldata = new IntObject(Integer.parseInt(lsdata));
        newTask.addIteration(ldata);
    }
    return newTask;
}

Example 19 with IntObject

use of org.apache.sysml.runtime.instructions.cp.IntObject in project systemml by apache.

the class Task method toFormatedString.

public String toFormatedString() {
    StringBuilder sb = new StringBuilder();
    sb.append("task (type=");
    sb.append(_type);
    sb.append(", iterations={");
    int count = 0;
    for (IntObject dat : _iterations) {
        if (count != 0)
            sb.append(";");
        sb.append("[");
        sb.append(_iterVar);
        sb.append("=");
        sb.append(dat.getLongValue());
        sb.append("]");
        count++;
    }
    sb.append("})");
    return sb.toString();
}

Example 20 with IntObject

use of org.apache.sysml.runtime.instructions.cp.IntObject in project systemml by apache.

the class Task method toCompactString.

public String toCompactString(int maxDigits) {
    StringBuilder sb = new StringBuilder();
    sb.append(_type);
    if (size() > 0) {
        sb.append(".");
        sb.append(_iterVar);
        sb.append(".{");
        int count = 0;
        for (IntObject dat : _iterations) {
            if (count != 0)
                sb.append(",");
            String tmp = String.valueOf(dat.getLongValue());
            for (int k = tmp.length(); k < maxDigits; k++) sb.append("0");
            sb.append(tmp);
            count++;
        }
        sb.append("}");
    }
    return sb.toString();
}