Examples with structures._ParentDoc4DCM structures._ParentDoc4DCM used on opensource projects

Example 36 with structures._ParentDoc4DCM

use of structures._ParentDoc4DCM in project IR_Base by Linda-sunshine.

the class weightedCorrespondenceModel method calculate_log_likelihood.

@Override
protected double calculate_log_likelihood() {
    double corpusLogLikelihood = 0;
    for (_Doc d : m_trainSet) {
        if (d instanceof _ParentDoc4DCM) {
            _ParentDoc4DCM pDoc = (_ParentDoc4DCM) d;
            corpusLogLikelihood += calculate_log_likelihood(pDoc);
        }
    }
    return corpusLogLikelihood;
}

Example 37 with structures._ParentDoc4DCM

use of structures._ParentDoc4DCM in project IR_Base by Linda-sunshine.

the class weightedCorrespondenceModel method collectStats.

protected void collectStats(_ParentDoc4DCM pDoc) {
    double gammaSum = Utils.sumOfArray(pDoc.m_sstat);
    for (int k = 0; k < number_of_topics; k++) {
        m_alpha_stat[k] += Utils.digamma(pDoc.m_sstat[k]) - Utils.digamma(gammaSum);
        double lambdaSum = Utils.sumOfArray(pDoc.m_lambda_stat[k]);
        for (int v = 0; v < vocabulary_size; v++) {
            m_beta_stat[k][v] += Utils.digamma(pDoc.m_lambda_stat[k][v]) - Utils.digamma(lambdaSum);
        }
    }
    for (_ChildDoc cDoc : pDoc.m_childDocs) {
        double piSum = Utils.sumOfArray(cDoc.m_sstat);
        for (int k = 0; k < number_of_topics; k++) m_alpha_c_stat[k] += Utils.digamma(cDoc.m_sstat[k]) - Utils.digamma(piSum);
    }
}

Example 38 with structures._ParentDoc4DCM

use of structures._ParentDoc4DCM in project IR_Base by Linda-sunshine.

the class weightedCorrespondenceModel method updatePi4Child.

public void updatePi4Child(_ParentDoc4DCM pDoc) {
    double gammaSum = Utils.sumOfArray(pDoc.m_sstat);
    for (_ChildDoc cDoc : pDoc.m_childDocs) {
        int[] iflag = { 0 }, iprint = { -1, 3 };
        double fValue = 0;
        int fSize = cDoc.m_sstat.length;
        double[] piGradient = new double[fSize];
        Arrays.fill(piGradient, 0);
        double[] piDiag = new double[fSize];
        Arrays.fill(piDiag, 0);
        double[] pi = new double[fSize];
        double[] oldPi = new double[fSize];
        for (int k = 0; k < fSize; k++) {
            pi[k] = Math.log(cDoc.m_sstat[k]);
            oldPi[k] = Math.log(cDoc.m_sstat[k]);
        }
        try {
            do {
                double diff = 0;
                fValue = piFuncGradientVal(pDoc, gammaSum, cDoc, pi, piGradient);
                LBFGS.lbfgs(fSize, 4, pi, fValue, piGradient, false, piDiag, iprint, 1e-2, 1e-10, iflag);
                for (int k = 0; k < fSize; k++) {
                    double tempDiff = 0;
                    tempDiff = pi[k] - oldPi[k];
                    if (Math.abs(tempDiff) > diff) {
                        diff = Math.abs(tempDiff);
                    }
                    oldPi[k] = pi[k];
                }
                if (diff < m_lbfgsConverge) {
                    // System.out.print("diff\t"+diff+"finish update pi");
                    break;
                }
            } while (iflag[0] != 0);
        } catch (LBFGS.ExceptionWithIflag e) {
            e.printStackTrace();
        }
        for (int k = 0; k < fSize; k++) {
            cDoc.m_sstat[k] = Math.exp(pi[k]);
        // System.out.println(cDoc.getName()+"\tcDoc.m_sstat[]"+cDoc.m_sstat[k]);
        }
    }
}

Example 39 with structures._ParentDoc4DCM

use of structures._ParentDoc4DCM in project IR_Base by Linda-sunshine.

the class weightedCorrespondenceModel method initialize_probability.

@Override
protected void initialize_probability(Collection<_Doc> collection) {
    init();
    for (_Doc d : collection) {
        if (d instanceof _ParentDoc4DCM) {
            int totalWords = 0;
            double totalLambda = 0;
            m_parentDocNum += 1;
            _ParentDoc4DCM pDoc = (_ParentDoc4DCM) d;
            pDoc.setTopics4Variational(number_of_topics, d_alpha, vocabulary_size, d_beta);
            totalWords += pDoc.getTotalDocLength();
            for (_Stn stnObj : pDoc.getSentences()) stnObj.setTopicsVct(number_of_topics);
            for (_ChildDoc cDoc : pDoc.m_childDocs) {
                totalWords += cDoc.getTotalDocLength();
                m_childDocNum += 1;
                cDoc.setTopics4Variational(number_of_topics, d_alpha);
                // update the article thread sufficient statistics
                for (int n = 0; n < cDoc.getSparse().length; n++) {
                    _SparseFeature fv = cDoc.getSparse()[n];
                    int wID = fv.getIndex();
                    double wVal = fv.getValue();
                    for (int k = 0; k < number_of_topics; k++) {
                        pDoc.m_lambda_stat[k][wID] += cDoc.m_phi[n][k] * wVal;
                    }
                }
            }
            for (int k = 0; k < number_of_topics; k++) {
                pDoc.m_lambda_topicStat[k] = Utils.sumOfArray(pDoc.m_lambda_stat[k]);
                totalLambda += pDoc.m_lambda_topicStat[k];
            }
        // System.out.println("totalWords\t"+totalWords+"\t"+totalLambda);
        }
    }
    imposePrior();
}

Example 40 with structures._ParentDoc4DCM

use of structures._ParentDoc4DCM in project IR_Base by Linda-sunshine.

the class weightedCorrespondenceModel method calculate_log_likelihood.

@Override
public double calculate_log_likelihood(_Doc d) {
    _ParentDoc4DCM pDoc = (_ParentDoc4DCM) d;
    double logLikelihood = 0;
    double gammaSum = Utils.sumOfArray(pDoc.m_sstat);
    double alphaSum = Utils.sumOfArray(m_alpha);
    logLikelihood += Utils.lgamma(alphaSum);
    logLikelihood -= Utils.lgamma(gammaSum);
    for (int k = 0; k < number_of_topics; k++) {
        logLikelihood += -Utils.lgamma(m_alpha[k]) + (m_alpha[k] - 1) * (Utils.digamma(pDoc.m_sstat[k]) - Utils.digamma(gammaSum));
        logLikelihood += Utils.lgamma(pDoc.m_sstat[k]);
        logLikelihood -= (pDoc.m_sstat[k] - 1) * (Utils.digamma(pDoc.m_sstat[k]) - Utils.digamma(gammaSum));
    }
    _SparseFeature[] fvs = pDoc.getSparse();
    for (int n = 0; n < fvs.length; n++) {
        int wID = fvs[n].getIndex();
        double wVal = fvs[n].getValue();
        for (int k = 0; k < number_of_topics; k++) {
            double updateLikelihood = 0;
            updateLikelihood -= pDoc.m_phi[n][k] * (Math.log(pDoc.m_phi[n][k]));
            if (Double.isInfinite(updateLikelihood)) {
                System.out.println("\nlikelihood\t" + updateLikelihood + "\t" + logLikelihood);
            }
            updateLikelihood += pDoc.m_phi[n][k] * (Utils.digamma(pDoc.m_sstat[k]) - Utils.digamma(gammaSum));
            if (Double.isInfinite(updateLikelihood)) {
                System.out.println("\nlikelihood\t" + updateLikelihood + "\t" + logLikelihood);
            }
            updateLikelihood += pDoc.m_phi[n][k] * wVal * (Utils.digamma(pDoc.m_lambda_stat[k][wID]) - Utils.digamma(pDoc.m_lambda_topicStat[k]));
            if (Double.isInfinite(updateLikelihood)) {
                System.out.println("\nlikelihood\t" + updateLikelihood + "\t" + logLikelihood);
                System.out.println("wVal\t" + wVal);
                System.out.println("pDoc.m_phi[n][k]\t" + pDoc.m_phi[n][k]);
                System.out.println("pDoc.m_phi[n][k]\t" + pDoc.m_phi[n][k]);
                System.out.println("pDoc.m_sstat[k]\t" + pDoc.m_sstat[k]);
                System.out.println("gammaSum\t" + gammaSum);
                System.out.println("pDoc.m_lambda_stat[k][wID]\t" + pDoc.m_lambda_stat[k][wID] + "\t" + Utils.digamma(pDoc.m_lambda_stat[k][wID]));
                System.out.println("pDoc.m_lambda_topicStat[k]\t" + pDoc.m_lambda_topicStat[k] + "\t" + Utils.digamma(pDoc.m_lambda_topicStat[k]));
            }
            logLikelihood += updateLikelihood;
            if (Double.isNaN(updateLikelihood)) {
                System.out.println("\nlikelihood\t" + updateLikelihood + "\t" + logLikelihood);
                System.out.println("wVal\t" + wVal);
                System.out.println("pDoc.m_phi[n][k]\t" + pDoc.m_phi[n][k]);
                System.out.println("pDoc.m_phi[n][k]\t" + pDoc.m_phi[n][k]);
                System.out.println("pDoc.m_sstat[k]\t" + pDoc.m_sstat[k]);
                System.out.println("gammaSum\t" + gammaSum);
                System.out.println("pDoc.m_lambda_stat[k][wID]\t" + pDoc.m_lambda_stat[k][wID]);
                System.out.println("pDoc.m_lambda_topicStat[k]\t" + pDoc.m_lambda_topicStat[k]);
            }
            if (Double.isInfinite(updateLikelihood)) {
                System.out.println("\nlikelihood\t" + updateLikelihood + "\t" + logLikelihood);
                System.out.println("wVal\t" + wVal);
                System.out.println("pDoc.m_phi[n][k]\t" + pDoc.m_phi[n][k]);
                System.out.println("pDoc.m_phi[n][k]\t" + pDoc.m_phi[n][k]);
                System.out.println("pDoc.m_sstat[k]\t" + pDoc.m_sstat[k]);
                System.out.println("gammaSum\t" + gammaSum);
                System.out.println("pDoc.m_lambda_stat[k][wID]\t" + pDoc.m_lambda_stat[k][wID]);
                System.out.println("pDoc.m_lambda_topicStat[k]\t" + pDoc.m_lambda_topicStat[k]);
            }
        }
    }
    double alphaCSum = Utils.sumOfArray(m_alpha_c);
    for (_ChildDoc cDoc : pDoc.m_childDocs) {
        logLikelihood += Utils.lgamma(alphaCSum);
        double piSum = Utils.sumOfArray(cDoc.m_sstat);
        logLikelihood -= Utils.lgamma(piSum);
        for (int k = 0; k < number_of_topics; k++) {
            logLikelihood -= Utils.lgamma(m_alpha_c[k]);
            logLikelihood += (m_alpha_c[k] - 1) * (Utils.digamma(cDoc.m_sstat[k]) - Utils.digamma(piSum));
            logLikelihood += Utils.lgamma(cDoc.m_sstat[k]);
            logLikelihood -= (cDoc.m_sstat[k] - 1) * (Utils.digamma(cDoc.m_sstat[k]) - Utils.digamma(piSum));
        }
        _SparseFeature[] cDocFvs = cDoc.getSparse();
        for (int n = 0; n < cDocFvs.length; n++) {
            int wID = cDocFvs[n].getIndex();
            double wVal = cDocFvs[n].getValue();
            for (int k = 0; k < number_of_topics; k++) {
                logLikelihood += cDoc.m_phi[n][k] * (Utils.digamma(pDoc.m_sstat[k]) - Utils.digamma(gammaSum) + Utils.digamma(cDoc.m_sstat[k]) - Utils.digamma(piSum));
                logLikelihood -= cDoc.m_phi[n][k] * (Utils.dotProduct(cDoc.m_sstat, pDoc.m_sstat) / (piSum * gammaSum * cDoc.m_zeta) + Math.log(cDoc.m_zeta) - 1);
                logLikelihood += wVal * cDoc.m_phi[n][k] * (Utils.digamma(pDoc.m_lambda_stat[k][wID]) - Utils.digamma(pDoc.m_lambda_topicStat[k]));
                logLikelihood -= cDoc.m_phi[n][k] * Math.log(cDoc.m_phi[n][k]);
                if (Double.isInfinite(logLikelihood)) {
                    System.out.println("\ncDoc likelihood\t" + "\t" + logLikelihood);
                    System.out.println("cDoc.m_phi[n][k]\t" + cDoc.m_phi[n][k]);
                    // System.out.println("pDoc.m_phi[n][k]\t"+pDoc.m_phi[n][k]);
                    System.out.println("pDoc.m_lambda_stat[][]\t" + pDoc.m_lambda_topicStat[k]);
                    System.out.println("cDoc.m_sstat[k]\t" + cDoc.m_sstat[k]);
                    System.out.println("piSum\t" + piSum);
                    // System.out.println("pDoc.m_lambda_stat[k][wID]\t" + pDoc.m_lambda_stat[k][wID]);
                    // System.out.println("pDoc.m_lambda_topicStat[k]\t" + pDoc.m_lambda_topicStat[k]);
                    System.out.println("cDoc zeta\t" + cDoc.m_zeta);
                }
            }
        }
    }
    for (int k = 0; k < number_of_topics; k++) {
        double betaSum = Utils.sumOfArray(m_beta[k]);
        logLikelihood += Utils.lgamma(betaSum);
        logLikelihood -= Utils.lgamma(pDoc.m_lambda_topicStat[k]);
        for (int v = 0; v < vocabulary_size; v++) {
            logLikelihood -= Utils.lgamma(m_beta[k][v]);
            logLikelihood += (m_beta[k][v] - 1) * (Utils.digamma(pDoc.m_lambda_stat[k][v]) - Utils.digamma(pDoc.m_lambda_topicStat[k]));
            logLikelihood += Utils.lgamma(pDoc.m_lambda_stat[k][v]);
            logLikelihood -= (pDoc.m_lambda_stat[k][v] - 1) * (Utils.digamma(pDoc.m_lambda_stat[k][v]) - Utils.digamma(pDoc.m_lambda_topicStat[k]));
        }
    }
    // System.out.println("doc \t"+pDoc.getName()+"\t likelihood \t"+logLikelihood);
    return logLikelihood;
}