Examples with TopicModelWorker topicmodels.multithreads.TopicModelWorker used on opensource projects

Example 1 with TopicModelWorker

use of topicmodels.multithreads.TopicModelWorker in project IR_Base by Linda-sunshine.

the class LDA_Gibbs_test method Evaluation.

public double Evaluation() {
    m_collectCorpusStats = false;
    double perplexity = 0, loglikelihood, log2 = Math.log(2.0), sumLikelihood = 0;
    double totalWords = 0.0;
    if (m_multithread) {
        multithread_inference();
        System.out.println("In thread");
        for (TopicModelWorker worker : m_workers) {
            sumLikelihood += worker.getLogLikelihood();
            perplexity += worker.getPerplexity();
        }
    } else {
        System.out.println("In Normal");
        for (_Doc d : m_testSet) {
            loglikelihood = inference(d);
            sumLikelihood += loglikelihood;
            perplexity += loglikelihood;
            totalWords += d.getDocTestLength();
        // perplexity += Math.pow(2.0,
        // -loglikelihood/d.getTotalDocLength() / log2);
        }
    }
    // perplexity /= m_testSet.size();
    perplexity /= totalWords;
    perplexity = Math.exp(-perplexity);
    sumLikelihood /= m_testSet.size();
    System.out.format("Test set perplexity is %.3f and log-likelihood is %.3f\n", perplexity, sumLikelihood);
    return perplexity;
}

Example 2 with TopicModelWorker

use of topicmodels.multithreads.TopicModelWorker in project IR_Base by Linda-sunshine.

the class TopicModel method Evaluation.

public double Evaluation() {
    m_collectCorpusStats = false;
    double perplexity = 0, loglikelihood, log2 = Math.log(2.0), sumLikelihood = 0;
    double totalWords = 0.0;
    if (m_multithread) {
        multithread_inference();
        System.out.println("In thread");
        for (TopicModelWorker worker : m_workers) {
            sumLikelihood += worker.getLogLikelihood();
            perplexity += worker.getPerplexity();
        }
    } else {
        System.out.println("In Normal");
        for (_Doc d : m_testSet) {
            loglikelihood = inference(d);
            sumLikelihood += loglikelihood;
            perplexity += loglikelihood;
            totalWords += d.getTotalDocLength();
        // perplexity += Math.pow(2.0, -loglikelihood/d.getTotalDocLength() / log2);
        }
    }
    // perplexity /= m_testSet.size();
    perplexity /= totalWords;
    perplexity = Math.exp(-perplexity);
    sumLikelihood /= m_testSet.size();
    if (this instanceof HTSM)
        calculatePrecisionRecall();
    System.out.format("Test set perplexity is %.3f and log-likelihood is %.3f\n", perplexity, sumLikelihood);
    return perplexity;
}

Example 3 with TopicModelWorker

use of topicmodels.multithreads.TopicModelWorker in project IR_Base by Linda-sunshine.

the class TopicModel method multithread_E_step.

double multithread_E_step() {
    for (int i = 0; i < m_workers.length; i++) {
        m_workers[i].setType(RunType.RT_EM);
        m_threadpool[i] = new Thread(m_workers[i]);
        m_threadpool[i].start();
    }
    // wait till all finished
    for (Thread thread : m_threadpool) {
        try {
            thread.join();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    double likelihood = 0;
    for (TopicModelWorker worker : m_workers) likelihood += worker.accumluateStats(word_topic_sstat);
    return likelihood;
}