Examples with structures._ParentDoc structures._ParentDoc used on opensource projects

Example 66 with structures._ParentDoc

use of structures._ParentDoc in project IR_Base by Linda-sunshine.

the class DCMCorrLDA method updateAlphaC.

protected void updateAlphaC() {
    double diff = 0;
    int iteration = 0;
    do {
        diff = 0;
        double totalAlphaDenominator = 0;
        double[] totalAlphaNumerator = new double[number_of_topics];
        Arrays.fill(totalAlphaNumerator, 0);
        m_totalAlpha_c = Utils.sumOfArray(m_alpha_c);
        double deltaAlpha = 0;
        for (_Doc d : m_trainSet) {
            if (d instanceof _ParentDoc) {
                _ParentDoc4DCM pDoc = (_ParentDoc4DCM) d;
                double pDocLen = pDoc.getTotalDocLength();
                for (_ChildDoc cDoc : pDoc.m_childDocs) {
                    double muDp = cDoc.getMu() / pDocLen;
                    double t_totalAlpha_c = m_totalAlpha_c + cDoc.getMu();
                    double digAlpha = Utils.digamma(t_totalAlpha_c);
                    totalAlphaDenominator += Utils.digamma(cDoc.getTotalDocLength() + t_totalAlpha_c) - digAlpha;
                    for (int k = 0; k < number_of_topics; k++) totalAlphaNumerator[k] += Utils.digamma(m_alpha_c[k] + muDp * pDoc.m_sstat[k] + cDoc.m_sstat[k]) - Utils.digamma(m_alpha_c[k] + muDp * pDoc.m_sstat[k]);
                }
            }
        }
        for (int k = 0; k < number_of_topics; k++) {
            deltaAlpha = totalAlphaNumerator[k] * 1.0 / totalAlphaDenominator;
            double newAlpha = m_alpha_c[k] * deltaAlpha;
            double t_diff = Math.abs(m_alpha_c[k] - newAlpha);
            if (t_diff > diff)
                diff = t_diff;
            m_alpha_c[k] = newAlpha;
        }
        iteration++;
        if (iteration > m_newtonIter)
            break;
    } while (diff > m_newtonConverge);
    // System.out.println("iteration\t" + iteration);
    m_totalAlpha_c = 0;
    for (int k = 0; k < number_of_topics; k++) {
        m_totalAlpha_c += m_alpha_c[k];
    }
}

Example 67 with structures._ParentDoc

use of structures._ParentDoc in project IR_Base by Linda-sunshine.

the class DCMCorrLDA_multi_E method initialize_probability.

protected void initialize_probability(Collection<_Doc> collection) {
    int cores = Runtime.getRuntime().availableProcessors();
    m_threadpool = new Thread[cores];
    m_workers = new DCMCorrLDA_worker[cores];
    for (int i = 0; i < cores; i++) m_workers[i] = new DCMCorrLDA_worker(number_of_topics, vocabulary_size);
    int workerID = 0;
    for (_Doc d : collection) {
        if (d instanceof _ParentDoc) {
            m_workers[workerID % cores].addDoc(d);
            _ParentDoc4DCM pDoc = (_ParentDoc4DCM) d;
            for (_ChildDoc cDoc : pDoc.m_childDocs) {
                m_workers[workerID % cores].addDoc(cDoc);
            }
            workerID++;
        }
    }
    super.initialize_probability(collection);
}

Example 68 with structures._ParentDoc

use of structures._ParentDoc in project IR_Base by Linda-sunshine.

the class DCMCorrLDA_multi_E_test method debugOutput.

protected void debugOutput(int topK, String filePrefix) {
    File parentTopicFolder = new File(filePrefix + "parentTopicAssignment");
    File childTopicFolder = new File(filePrefix + "childTopicAssignment");
    if (!parentTopicFolder.exists()) {
        System.out.println("creating directory" + parentTopicFolder);
        parentTopicFolder.mkdir();
    }
    if (!childTopicFolder.exists()) {
        System.out.println("creating directory" + childTopicFolder);
        childTopicFolder.mkdir();
    }
    File parentWordTopicDistributionFolder = new File(filePrefix + "wordTopicDistribution");
    if (!parentWordTopicDistributionFolder.exists()) {
        System.out.println("creating word topic distribution folder\t" + parentWordTopicDistributionFolder);
        parentWordTopicDistributionFolder.mkdir();
    }
    for (_Doc d : m_trainSet) {
        if (d instanceof _ParentDoc) {
            // printParentTopicAssignment(d, parentTopicFolder);
            printWordTopicDistribution(d, parentWordTopicDistributionFolder, topK);
        } else {
            printChildTopicAssignment(d, childTopicFolder);
        }
    }
    String parentParameterFile = filePrefix + "parentParameter.txt";
    String childParameterFile = filePrefix + "childParameter.txt";
    printParameter(parentParameterFile, childParameterFile, m_trainSet);
    printTopKChild4Stn(filePrefix, topK);
}

Example 69 with structures._ParentDoc

use of structures._ParentDoc in project IR_Base by Linda-sunshine.

the class DCMCorrLDA_test method printParameter.

protected void printParameter(String parentParameterFile, String childParameterFile, ArrayList<_Doc> docList) {
    System.out.println("printing parameter");
    try {
        System.out.println(parentParameterFile);
        System.out.println(childParameterFile);
        PrintWriter parentParaOut = new PrintWriter(new File(parentParameterFile));
        PrintWriter childParaOut = new PrintWriter(new File(childParameterFile));
        for (_Doc d : docList) {
            if (d instanceof _ParentDoc) {
                parentParaOut.print(d.getName() + "\t");
                parentParaOut.print("topicProportion\t");
                for (int k = 0; k < number_of_topics; k++) {
                    parentParaOut.print(d.m_topics[k] + "\t");
                }
                parentParaOut.println();
                for (_ChildDoc cDoc : ((_ParentDoc) d).m_childDocs) {
                    childParaOut.print(cDoc.getName() + "\t");
                    childParaOut.print("topicProportion\t");
                    for (int k = 0; k < number_of_topics; k++) {
                        childParaOut.print(cDoc.m_topics[k] + "\t");
                    }
                    childParaOut.println();
                }
            }
        }
        parentParaOut.flush();
        parentParaOut.close();
        childParaOut.flush();
        childParaOut.close();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 70 with structures._ParentDoc

use of structures._ParentDoc in project IR_Base by Linda-sunshine.

the class ParentChildAnalyzer method loadParentDoc.

public void loadParentDoc(String fileName) {
    if (fileName == null || fileName.isEmpty())
        return;
    JSONObject json = LoadJSON(fileName);
    String title = Utils.getJSONValue(json, "title");
    String content = Utils.getJSONValue(json, "content");
    String name = Utils.getJSONValue(json, "name");
    String[] sentences = null;
    // _ParentDoc d = new _ParentDoc(m_corpus.getSize(), name, title,
    // content, 0);
    _ParentDoc d = new _ParentDoc4DCM(m_corpus.getSize(), name, title, content, 0);
    try {
        JSONArray sentenceArray = json.getJSONArray("sentences");
        sentences = new String[sentenceArray.length()];
        // shall we add title into this sentence array
        for (int i = 0; i < sentenceArray.length(); i++) sentences[i] = Utils.getJSONValue(sentenceArray.getJSONObject(i), "sentence");
        if (AnalyzeDocByStn(d, sentences))
            parentHashMap.put(name, d);
    } catch (JSONException e) {
        e.printStackTrace();
    }
}