Examples with H2OCountedCompleter water.H2O.H2OCountedCompleter used on opensource projects

Example 11 with H2OCountedCompleter

use of water.H2O.H2OCountedCompleter in project h2o-2 by h2oai.

the class GLM2 method fork.

public GLM2 fork(H2OCountedCompleter cc) {
    if (!_grid)
        source.read_lock(self());
    // keep *this* separate from what's stored in K/V as job (will be changing it!)
    Futures fs = new Futures();
    _progressKey = Key.make(dest().toString() + "_progress", (byte) 1, Key.HIDDEN_USER_KEY, dest().home_node());
    int total = max_iter;
    if (lambda_search)
        total = MAX_ITERATIONS_PER_LAMBDA * nlambdas;
    GLM2_Progress progress = new GLM2_Progress(total * (n_folds > 1 ? (n_folds + 1) : 1));
    LogInfo("created progress " + progress);
    DKV.put(_progressKey, progress, fs);
    fs.blockForPending();
    _fjtask = new H2O.H2OEmptyCompleter(cc);
    H2OCountedCompleter fjtask = new GLMJobCompleter(_fjtask);
    GLM2 j = (GLM2) clone();
    // modifying GLM2 object, don't want job object to be the same instance
    j.start(_fjtask);
    H2O.submitTask(fjtask);
    return j;
}

Example 12 with H2OCountedCompleter

use of water.H2O.H2OCountedCompleter in project h2o-2 by h2oai.

the class Frame method vecs_impl.

// Compute vectors for caching
private Vec[] vecs_impl() {
    // Load all Vec headers; load them all in parallel by spawning F/J tasks.
    final Vec[] vecs = new Vec[_keys.length];
    Futures fs = new Futures();
    for (int i = 0; i < _keys.length; i++) {
        final int ii = i;
        final Key k = _keys[i];
        H2OCountedCompleter t = new H2OCountedCompleter() {

            // We need higher priority here as there is a danger of deadlock in
            // case of many calls from MRTask2 at once (e.g. frame with many
            // vectors invokes rollup tasks for all vectors in parallel).  Should
            // probably be done in CPS style in the future
            @Override
            public byte priority() {
                return H2O.MIN_HI_PRIORITY;
            }

            @Override
            public void compute2() {
                Value v = DKV.get(k);
                if (v == null)
                    Log.err("Missing vector #" + ii + " (" + _names[ii] + ") during Frame fetch: " + k);
                vecs[ii] = v.get();
                tryComplete();
            }
        };
        H2O.submitTask(t);
        fs.add(t);
    }
    fs.blockForPending();
    return vecs;
}

Example 13 with H2OCountedCompleter

use of water.H2O.H2OCountedCompleter in project h2o-2 by h2oai.

the class ParseDataset2 method forkParseDataset.

// Same parse, as a backgroundable Job
public static ParseDataset2 forkParseDataset(final Key dest, Key[] keys, final CustomParser.ParserSetup setup, boolean delete_on_done) {
    keys = filterEmptyFiles(keys);
    setup.checkDupColumnNames();
    // Some quick sanity checks: no overwriting your input key, and a resource check.
    long sum = 0;
    for (Key k : keys) {
        if (dest.equals(k))
            throw new IllegalArgumentException("Destination key " + dest + " must be different from all sources");
        // Sum of all input filesizes
        sum += DKV.get(k).length();
    }
    // Cluster memory
    long memsz = 0;
    for (H2ONode h2o : H2O.CLOUD._memary) memsz += h2o.get_max_mem();
    if (sum > memsz * 4)
        throw new IllegalArgumentException("Total input file size of " + PrettyPrint.bytes(sum) + " is much larger than total cluster memory of " + PrettyPrint.bytes(memsz) + ", please use either a larger cluster or smaller data.");
    ParseDataset2 job = new ParseDataset2(dest, keys);
    // Lock BEFORE returning
    new Frame(job.dest(), new String[0], new Vec[0]).delete_and_lock(job.self());
    // Lock BEFORE returning
    for (Key k : keys) Lockable.read_lock(k, job.self());
    // Fire off background parse
    ParserFJTask fjt = new ParserFJTask(job, keys, setup, delete_on_done);
    // Make a wrapper class that only *starts* when the ParserFJTask fjt
    // completes - especially it only starts even when fjt completes
    // exceptionally... thus the fjt onExceptionalCompletion code runs
    // completely before this empty task starts - providing a simple barrier.
    // Threads blocking on the job will block on the "cleanup" task, which will
    // block until the fjt runs the onCompletion or onExceptionCompletion code.
    H2OCountedCompleter cleanup = new H2OCountedCompleter() {

        @Override
        public void compute2() {
        }

        @Override
        public boolean onExceptionalCompletion(Throwable ex, CountedCompleter caller) {
            return true;
        }
    };
    fjt.setCompleter(cleanup);
    job.start(cleanup);
    H2O.submitTask(fjt);
    return job;
}

Example 14 with H2OCountedCompleter

use of water.H2O.H2OCountedCompleter in project h2o-2 by h2oai.

the class FrameExtractor method compute2.

@Override
public void compute2() {
    // Lock all possible data
    dataset.read_lock(jobKey);
    // Create a template vector for each segment
    final Vec[][] templates = makeTemplates();
    final int nsplits = templates.length;
    assert templates.length == numOfOutputs() : "Number of outputs and number of created templates differ!";
    final Vec[] datasetVecs = dataset.vecs();
    // Create output frames
    splits = new Frame[nsplits];
    for (int s = 0; s < nsplits; s++) {
        Frame split = new Frame(destKeys[s], dataset.names(), templates[s]);
        split.delete_and_lock(jobKey);
        splits[s] = split;
    }
    // Launch number of distributed FJ for each split part
    setPendingCount(1);
    H2O.submitTask(new H2OCountedCompleter(FrameExtractor.this) {

        @Override
        public void compute2() {
            setPendingCount(nsplits);
            for (int s = 0; s < nsplits; s++) {
                MRTask2 mrt = createNewWorker(new // Completer for this task
                H2OCountedCompleter(// Completer for this task
                this) {

                    @Override
                    public void compute2() {
                    }

                    @Override
                    public boolean onExceptionalCompletion(Throwable ex, CountedCompleter caller) {
                        synchronized (FrameExtractor.this) {
                            // synchronized on this since can be accessed from different workers
                            workersExceptions = workersExceptions != null ? Arrays.copyOf(workersExceptions, workersExceptions.length + 1) : new Throwable[1];
                            workersExceptions[workersExceptions.length - 1] = ex;
                        }
                        // we handle the exception so wait perform normal completion
                        tryComplete();
                        return false;
                    }
                }, datasetVecs, s);
                assert mrt.getCompleter() != null : "The `createNewWorker` method violates API contract and forgets to setup given counted completer!";
                mrt.asyncExec(splits[s]);
            }
            // complete the computation of nsplits-tasks
            tryComplete();
        }
    });
    // complete the computation of thrown tasks
    tryComplete();
}