Examples with structures._ParentDoc4DCM structures._ParentDoc4DCM used on opensource projects

Example 31 with structures._ParentDoc4DCM

use of structures._ParentDoc4DCM in project IR_Base by Linda-sunshine.

the class DCMLDA4AC method cal_logLikelihood_partial4Child.

protected double cal_logLikelihood_partial4Child(_ChildDoc d) {
    double likelihood = 0;
    _ParentDoc4DCM pDoc = (_ParentDoc4DCM) d.m_parentDoc;
    for (_Word w : d.getWords()) {
        int wid = w.getIndex();
        double wordLikelihood = 0;
        for (int k = 0; k < number_of_topics; k++) {
            wordLikelihood += d.m_topics[k] * pDoc.m_wordTopic_prob[k][wid];
        }
        likelihood += Math.log(wordLikelihood);
    }
    return likelihood;
}

Example 32 with structures._ParentDoc4DCM

use of structures._ParentDoc4DCM in project IR_Base by Linda-sunshine.

the class DCMLDA4AC method initialAlphaBeta.

protected void initialAlphaBeta() {
    Arrays.fill(m_sstat, 0);
    for (int k = 0; k < number_of_topics; k++) {
        Arrays.fill(topic_term_probabilty[k], 0);
    }
    for (_Doc d : m_trainSet) {
        if (d instanceof _ParentDoc4DCM) {
            _ParentDoc4DCM pDoc = (_ParentDoc4DCM) d;
            for (int k = 0; k < number_of_topics; k++) {
                double tempProb = pDoc.m_sstat[k] / pDoc.getTotalDocLength();
                m_sstat[k] += tempProb;
                if (pDoc.m_sstat[k] == 0)
                    continue;
                for (int v = 0; v < vocabulary_size; v++) {
                    tempProb = pDoc.m_wordTopic_prob[k][v] / pDoc.m_sstat[k];
                    topic_term_probabilty[k][v] += tempProb;
                }
            }
            for (_ChildDoc cDoc : pDoc.m_childDocs) {
                for (int k = 0; k < number_of_topics; k++) {
                    double tempProb = cDoc.m_sstat[k] / cDoc.getTotalDocLength();
                    m_sstat[k] += tempProb;
                }
            }
        }
    }
    int trainSetSize = m_trainSet.size();
    for (int k = 0; k < number_of_topics; k++) {
        m_sstat[k] /= trainSetSize;
        for (int v = 0; v < vocabulary_size; v++) {
            topic_term_probabilty[k][v] /= trainSetSize;
        }
    }
    for (int k = 0; k < number_of_topics; k++) {
        m_alpha[k] = m_sstat[k];
        for (int v = 0; v < vocabulary_size; v++) {
            m_beta[k][v] = topic_term_probabilty[k][v] + d_beta;
        }
    }
    m_totalAlpha = Utils.sumOfArray(m_alpha);
    for (int k = 0; k < number_of_topics; k++) {
        m_totalBeta[k] = Utils.sumOfArray(m_beta[k]);
    }
}

Example 33 with structures._ParentDoc4DCM

use of structures._ParentDoc4DCM in project IR_Base by Linda-sunshine.

the class DCMLDA4AC method initTestDoc.

public void initTestDoc(ArrayList<_Doc> sampleTestSet, _Doc d) {
    _ParentDoc4DCM pDoc = (_ParentDoc4DCM) d;
    for (_Stn stnObj : pDoc.getSentences()) {
        stnObj.setTopicsVct(number_of_topics);
    }
    int testLength = 0;
    pDoc.setTopics4GibbsTest(number_of_topics, 0, testLength, vocabulary_size);
    sampleTestSet.add(pDoc);
    for (_ChildDoc cDoc : pDoc.m_childDocs) {
        testLength = (int) (m_testWord4PerplexityProportion * cDoc.getTotalDocLength());
        cDoc.setTopics4GibbsTest(number_of_topics, d_alpha, testLength);
        for (_Word w : d.getWords()) {
            int wid = w.getIndex();
            int tid = w.getTopic();
            pDoc.m_wordTopic_stat[tid][wid]++;
            pDoc.m_topic_stat[tid]++;
        }
        sampleTestSet.add(cDoc);
        cDoc.createSparseVct4Infer();
    }
}

Example 34 with structures._ParentDoc4DCM

use of structures._ParentDoc4DCM in project IR_Base by Linda-sunshine.

the class DCMLDA4AC_test method debugOutput.

public void debugOutput(int topK, String filePrefix) {
    File parentTopicFolder = new File(filePrefix + "parentTopicAssignment");
    File childTopicFolder = new File(filePrefix + "childTopicAssignment");
    if (!parentTopicFolder.exists()) {
        System.out.println("creating directory\t" + parentTopicFolder);
        parentTopicFolder.mkdir();
    }
    if (!childTopicFolder.exists()) {
        System.out.println("creating directory\t" + childTopicFolder);
        childTopicFolder.mkdir();
    }
    File parentWordTopicDistributionFolder = new File(filePrefix + "wordTopicDistribution");
    if (!parentWordTopicDistributionFolder.exists()) {
        System.out.println("creating word topic distribution folder\t" + parentWordTopicDistributionFolder);
        parentWordTopicDistributionFolder.mkdir();
    }
    for (_Doc d : m_trainSet) {
        if (d instanceof _ParentDoc4DCM) {
            printParentTopicAssignment(d, parentTopicFolder);
            printWordTopicDistribution(d, parentWordTopicDistributionFolder, topK);
        } else {
            printChildTopicAssignment(d, childTopicFolder);
        }
    }
    String parentParameterFile = filePrefix + "parentParameter.txt";
    String childParameterFile = filePrefix + "childParameter.txt";
    printParameter(parentParameterFile, childParameterFile, m_trainSet);
    String betaFile = filePrefix + "/topBetas.txt";
    printTopBeta(topK, betaFile);
}

Example 35 with structures._ParentDoc4DCM

use of structures._ParentDoc4DCM in project IR_Base by Linda-sunshine.

the class ParentChildAnalyzer method loadParentDoc.

public void loadParentDoc(String fileName) {
    if (fileName == null || fileName.isEmpty())
        return;
    JSONObject json = LoadJSON(fileName);
    String title = Utils.getJSONValue(json, "title");
    String content = Utils.getJSONValue(json, "content");
    String name = Utils.getJSONValue(json, "name");
    String[] sentences = null;
    // _ParentDoc d = new _ParentDoc(m_corpus.getSize(), name, title,
    // content, 0);
    _ParentDoc d = new _ParentDoc4DCM(m_corpus.getSize(), name, title, content, 0);
    try {
        JSONArray sentenceArray = json.getJSONArray("sentences");
        sentences = new String[sentenceArray.length()];
        // shall we add title into this sentence array
        for (int i = 0; i < sentenceArray.length(); i++) sentences[i] = Utils.getJSONValue(sentenceArray.getJSONObject(i), "sentence");
        if (AnalyzeDocByStn(d, sentences))
            parentHashMap.put(name, d);
    } catch (JSONException e) {
        e.printStackTrace();
    }
}