Examples with NamedThreadFactory edu.cmu.ml.proppr.util.multithreading.NamedThreadFactory used on opensource projects

Example 1 with NamedThreadFactory

use of edu.cmu.ml.proppr.util.multithreading.NamedThreadFactory in project ProPPR by TeamCohen.

the class CachingTrainer method trainCached.

public ParamVector<String, ?> trainCached(List<PosNegRWExample> examples, LearningGraphBuilder builder, ParamVector<String, ?> initialParamVec, int numEpochs, TrainingStatistics total) {
    ParamVector<String, ?> paramVec = this.masterLearner.setupParams(initialParamVec);
    NamedThreadFactory trainThreads = new NamedThreadFactory("work-");
    ExecutorService trainPool;
    ExecutorService cleanPool;
    StoppingCriterion stopper = new StoppingCriterion(numEpochs, this.stoppingPercent, this.stoppingEpoch);
    boolean graphSizesStatusLog = true;
    // repeat until ready to stop
    while (!stopper.satisified()) {
        // set up current epoch
        this.epoch++;
        for (SRW learner : this.learners.values()) {
            learner.setEpoch(epoch);
            learner.clearLoss();
        }
        log.info("epoch " + epoch + " ...");
        status.tick();
        // reset counters & file pointers
        this.statistics = new TrainingStatistics();
        trainThreads.reset();
        trainPool = Executors.newFixedThreadPool(this.nthreads, trainThreads);
        cleanPool = Executors.newSingleThreadExecutor();
        // run examples
        int id = 1;
        if (this.shuffle)
            Collections.shuffle(examples);
        for (PosNegRWExample s : examples) {
            Future<ExampleStats> trained = trainPool.submit(new Train(new PretendParse(s), paramVec, id, null));
            cleanPool.submit(new TraceLosses(trained, id));
            id++;
            if (log.isInfoEnabled() && status.due(1))
                log.info("queued: " + id + " trained: " + statistics.exampleSetSize);
        }
        cleanEpoch(trainPool, cleanPool, paramVec, stopper, id, total);
        if (graphSizesStatusLog) {
            log.info("Dataset size stats: " + statistics.totalGraphSize + " total nodes / max " + statistics.maxGraphSize + " / avg " + (statistics.totalGraphSize / id));
            graphSizesStatusLog = false;
        }
    }
    log.info("Reading: " + total.readTime + " Parsing: " + total.parseTime + " Training: " + total.trainTime);
    return paramVec;
}

Example 2 with NamedThreadFactory

use of edu.cmu.ml.proppr.util.multithreading.NamedThreadFactory in project ProPPR by TeamCohen.

the class Trainer method findGradient.

public ParamVector<String, ?> findGradient(SymbolTable<String> masterFeatures, Iterable<String> examples, LearningGraphBuilder builder, ParamVector<String, ?> paramVec) {
    log.info("Computing gradient on cooked examples...");
    ParamVector<String, ?> sumGradient = new SimpleParamVector<String>();
    if (paramVec == null) {
        paramVec = createParamVector();
    }
    paramVec = this.masterLearner.setupParams(paramVec);
    if (masterFeatures != null && masterFeatures.size() > 0)
        LearningGraphBuilder.setFeatures(masterFeatures);
    //		
    //		//WW: accumulate example-size normalized gradient
    //		for (PosNegRWExample x : examples) {
    ////			this.learner.initializeFeatures(paramVec,x.getGraph());
    //			this.learner.accumulateGradient(paramVec, x, sumGradient);
    //			k++;
    //		}
    NamedThreadFactory workThreads = new NamedThreadFactory("work-");
    ExecutorService workPool, cleanPool;
    workPool = Executors.newFixedThreadPool(this.nthreads, workThreads);
    cleanPool = Executors.newSingleThreadExecutor();
    // run examples
    int id = 1;
    int countdown = -1;
    Trainer notify = null;
    status.start();
    for (String s : examples) {
        if (log.isInfoEnabled() && status.due())
            log.info(id + " examples read...");
        long queueSize = (((ThreadPoolExecutor) workPool).getTaskCount() - ((ThreadPoolExecutor) workPool).getCompletedTaskCount());
        if (log.isDebugEnabled())
            log.debug("Queue size " + queueSize);
        if (countdown > 0) {
            if (log.isDebugEnabled())
                log.debug("Countdown " + countdown);
            countdown--;
        } else if (countdown == 0) {
            if (log.isDebugEnabled())
                log.debug("Countdown " + countdown + "; throttling:");
            countdown--;
            notify = null;
            try {
                synchronized (this) {
                    if (log.isDebugEnabled())
                        log.debug("Clearing training queue...");
                    while ((((ThreadPoolExecutor) workPool).getTaskCount() - ((ThreadPoolExecutor) workPool).getCompletedTaskCount()) > this.nthreads) this.wait();
                    if (log.isDebugEnabled())
                        log.debug("Queue cleared.");
                }
            } catch (InterruptedException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        } else if (queueSize > 1.5 * this.nthreads) {
            if (log.isDebugEnabled())
                log.debug("Starting countdown");
            countdown = this.nthreads;
            notify = this;
        }
        Future<PosNegRWExample> parsed = workPool.submit(new Parse(s, builder, id));
        Future<ExampleStats> gradfound = workPool.submit(new Grad(parsed, paramVec, sumGradient, id, notify));
        cleanPool.submit(new TraceLosses(gradfound, id));
        id++;
    }
    workPool.shutdown();
    try {
        workPool.awaitTermination(7, TimeUnit.DAYS);
        cleanPool.shutdown();
        cleanPool.awaitTermination(7, TimeUnit.DAYS);
    } catch (InterruptedException e) {
        log.error("Interrupted?", e);
    }
    this.masterLearner.cleanupParams(paramVec, sumGradient);
    //WW: renormalize by the total number of queries
    for (Iterator<String> it = sumGradient.keySet().iterator(); it.hasNext(); ) {
        String feature = it.next();
        double unnormf = sumGradient.get(feature);
        // query count stored in numExamplesThisEpoch, as noted above
        double norm = unnormf / this.statistics.numExamplesThisEpoch;
        sumGradient.put(feature, norm);
    }
    return sumGradient;
}

Example 3 with NamedThreadFactory

use of edu.cmu.ml.proppr.util.multithreading.NamedThreadFactory in project ProPPR by TeamCohen.

the class Trainer method train.

public ParamVector<String, ?> train(SymbolTable<String> masterFeatures, Iterable<String> examples, LearningGraphBuilder builder, ParamVector<String, ?> initialParamVec, int numEpochs) {
    ParamVector<String, ?> paramVec = this.masterLearner.setupParams(initialParamVec);
    if (masterFeatures.size() > 0)
        LearningGraphBuilder.setFeatures(masterFeatures);
    NamedThreadFactory workingThreads = new NamedThreadFactory("work-");
    NamedThreadFactory cleaningThreads = new NamedThreadFactory("cleanup-");
    ThreadPoolExecutor workingPool;
    ExecutorService cleanPool;
    TrainingStatistics total = new TrainingStatistics();
    StoppingCriterion stopper = new StoppingCriterion(numEpochs, this.stoppingPercent, this.stoppingEpoch);
    boolean graphSizesStatusLog = true;
    StatusLogger stattime = new StatusLogger();
    // repeat until ready to stop
    while (!stopper.satisified()) {
        // set up current epoch
        this.epoch++;
        for (SRW learner : this.learners.values()) {
            learner.setEpoch(epoch);
            learner.clearLoss();
        }
        log.info("epoch " + epoch + " ...");
        status.tick();
        // reset counters & file pointers
        this.statistics = new TrainingStatistics();
        workingThreads.reset();
        cleaningThreads.reset();
        workingPool = new ThreadPoolExecutor(this.nthreads, Integer.MAX_VALUE, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), workingThreads);
        cleanPool = Executors.newSingleThreadExecutor(cleaningThreads);
        // run examples
        int id = 1;
        stattime.start();
        int countdown = -1;
        Trainer notify = null;
        for (String s : examples) {
            if (log.isDebugEnabled())
                log.debug("Queue size " + (workingPool.getTaskCount() - workingPool.getCompletedTaskCount()));
            statistics.updateReadingStatistics(stattime.sinceLast());
            /*
				 * Throttling behavior:
				 * Once the number of unfinished tasks exceeds 1.5x the number of threads,
				 * we add a 'notify' object to the next nthreads training tasks. Then, the
				 * master thread gathers 'notify' signals until the number of unfinished tasks 
				 * is no longer greater than the number of threads. Then we start adding tasks again.
				 * 
				 * This works more or less fine, since the master thread stops pulling examples
				 * from disk when there are then a maximum of 2.5x training examples in the queue (that's
				 * the original 1.5x, which could represent a maximum of 1.5x training examples,
				 * plus the nthreads training tasks with active 'notify' objects. There's an 
				 * additional nthreads parsing tasks in the queue but those don't take up much 
				 * memory so we don't care). This lets us read in a good-sized buffer without
				 * blowing up the heap.
				 * 
				 * Worst-case: None of the backlog is cleared before the master thread enters
				 * the synchronized block. nthreads-1 threads will be training long jobs, and 
				 * the one free thread works through the 0.5x backlog and all nthreads countdown 
				 * examples. The notify() sent by the final countdown example will occur when 
				 * there are nthreads unfinished tasks in the queue, and the master thread will exit
				 * the synchronized block and proceed.
				 * 
				 * Best-case: The backlog is already cleared by the time the master thread enters
				 * the synchronized block. The while() loop immediately exits, and the notify()
				 * signals from the countdown examples have no effect.
				 */
            if (countdown > 0) {
                if (log.isDebugEnabled())
                    log.debug("Countdown " + countdown);
                countdown--;
            } else if (countdown == 0) {
                if (log.isDebugEnabled())
                    log.debug("Countdown " + countdown + "; throttling:");
                countdown--;
                notify = null;
                try {
                    synchronized (this) {
                        if (log.isDebugEnabled())
                            log.debug("Clearing training queue...");
                        while (workingPool.getTaskCount() - workingPool.getCompletedTaskCount() > this.nthreads) this.wait();
                        if (log.isDebugEnabled())
                            log.debug("Queue cleared.");
                    }
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            } else if (workingPool.getTaskCount() - workingPool.getCompletedTaskCount() > 1.5 * this.nthreads) {
                if (log.isDebugEnabled())
                    log.debug("Starting countdown");
                countdown = this.nthreads;
                notify = this;
            }
            Future<PosNegRWExample> parsed = workingPool.submit(new Parse(s, builder, id));
            Future<ExampleStats> trained = workingPool.submit(new Train(parsed, paramVec, id, notify));
            cleanPool.submit(new TraceLosses(trained, id));
            id++;
            stattime.tick();
            if (log.isInfoEnabled() && status.due(1))
                log.info("parsed: " + id + " trained: " + statistics.exampleSetSize);
        }
        cleanEpoch(workingPool, cleanPool, paramVec, stopper, id, total);
        if (graphSizesStatusLog) {
            log.info("Dataset size stats: " + statistics.totalGraphSize + " total nodes / max " + statistics.maxGraphSize + " / avg " + (statistics.totalGraphSize / id));
            graphSizesStatusLog = false;
        }
    }
    log.info("Reading  statistics: min " + total.minReadTime + " / max " + total.maxReadTime + " / total " + total.readTime);
    log.info("Parsing  statistics: min " + total.minParseTime + " / max " + total.maxParseTime + " / total " + total.parseTime);
    log.info("Training statistics: min " + total.minTrainTime + " / max " + total.maxTrainTime + " / total " + total.trainTime);
    return paramVec;
}