Examples with TailLOUDSTrie org.trie4j.louds.TailLOUDSTrie used on opensource projects

Example 1 with TailLOUDSTrie

use of org.trie4j.louds.TailLOUDSTrie in project trie4j by takawitter.

the class AbstractWikipediaSerializeTest method getBvTreeClassName.

static String getBvTreeClassName(Trie trie) {
    if (trie instanceof TailLOUDSTrie) {
        StringBuilder b = new StringBuilder("(");
        BvTree bvTree = ((TailLOUDSTrie) trie).getBvTree();
        b.append(bvTree.getClass().getSimpleName());
        if (bvTree instanceof LOUDSBvTree) {
            b.append("(");
            b.append(((LOUDSBvTree) bvTree).getSbv().getClass().getSimpleName());
            b.append(")");
        } else if (bvTree instanceof LOUDSPPBvTree) {
            b.append("(");
            LOUDSPPBvTree pbvt = (LOUDSPPBvTree) bvTree;
            b.append("r0:").append(pbvt.getR0().getClass().getSimpleName()).append(",r1:").append(pbvt.getR1().getClass().getSimpleName()).append(")");
        }
        b.append(")");
        return b.toString();
    } else {
        return "";
    }
}

Example 2 with TailLOUDSTrie

use of org.trie4j.louds.TailLOUDSTrie in project trie4j by takawitter.

the class TrieWriterTest method test.

@Test
public void test() throws Exception {
    LapTimer lt = new LapTimer();
    PatriciaTrie origTrie = new PatriciaTrie();
    new WikipediaTitles().insertTo(origTrie);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    TrieWriter tw = new TrieWriter(baos);
    Trie trie = new TailLOUDSTrie(origTrie, new LOUDSPPBvTree(origTrie.nodeSize()), new SuffixTrieDenseTailArrayBuilder());
    lt.reset();
    tw.write(trie);
    tw.flush();
    lt.lapMillis("trie saved.");
    System.out.println(baos.size() + " bytes");
    TrieReader tr = new TrieReader(new ByteArrayInputStream(baos.toByteArray()));
    lt.reset();
    Trie trie2 = tr.read();
    lt.lapMillis("trie loaded.");
    long d = new WikipediaTitles().assertAllContains(trie2);
    System.out.println("[" + d + "ms]: verified");
}

Example 3 with TailLOUDSTrie

use of org.trie4j.louds.TailLOUDSTrie in project trie4j by takawitter.

the class SBVConcatTailArrayTest method test.

@Test
public void test() throws Exception {
    // 普通にSBVConcatTailArrayIndexBuilder使った場合と、
    // add毎にappendするTailArrayIndexBuilderを作ってそれを使った
    // 場合でbitvectorやcacheに差が出るか調べる
    TailPatriciaTrie org = new TailPatriciaTrie(new ConcatTailBuilder());
    new WikipediaTitles().insertTo(org);
    TailLOUDSTrie louds1 = new TailLOUDSTrie(org, new SBVConcatTailArrayAppendingBuilder());
    new WikipediaTitles().assertAllContains(louds1);
    BytesSuccinctBitVector sbv1 = (BytesSuccinctBitVector) ((SBVTailIndex) ((DefaultTailArray) louds1.getTailArray()).getTailIndex()).getSbv();
    TailLOUDSTrie louds2 = new TailLOUDSTrie(org, new SBVConcatTailArrayBuilder());
    new WikipediaTitles().assertAllContains(louds2);
    BytesSuccinctBitVector sbv2 = (BytesSuccinctBitVector) ((SBVTailIndex) ((DefaultTailArray) louds2.getTailArray()).getTailIndex()).getSbv();
    {
        int n = sbv1.size();
        System.out.println("sbv size: " + n);
        Assert.assertEquals(n, sbv2.size());
        for (int i = 0; i < n; i++) {
            Assert.assertEquals(i + "th bit", sbv1.get(i), sbv2.get(i));
        }
    }
    {
        int[] countCache1 = sbv1.getCountCache0();
        int[] countCache2 = sbv2.getCountCache0();
        int n = countCache1.length;
        System.out.println("countCache0 size should be: " + (sbv1.size() / 64 + 1));
        System.out.println("countCache0 size: " + n);
        //			Assert.assertEquals(n, countCache2.length);
        n = Math.min(countCache1.length, countCache2.length);
        for (int i = 0; i < n; i++) {
            Assert.assertEquals(i + "th index cache.", countCache1[i], countCache2[i]);
        }
    }
    {
        IntArray indexCache1 = sbv1.getIndexCache0();
        IntArray indexCache2 = sbv2.getIndexCache0();
        int n = indexCache1.size();
        System.out.println("indexCache0 size1: " + n);
        System.out.println("indexCache0 size2: " + indexCache2.size());
        //			Assert.assertEquals(n, countCache2.length);
        n = Math.min(indexCache1.size(), indexCache2.size());
        for (int i = 0; i < 10; i++) {
            System.out.print(indexCache1.get(i) + ", ");
        }
        System.out.println();
        for (int i = 0; i < 10; i++) {
            System.out.print(indexCache2.get(i) + ", ");
        }
        System.out.println();
        for (int i = 0; i < n; i++) {
            Assert.assertEquals(i + "th index cache.", indexCache1.get(i), indexCache2.get(i));
        }
    }
}

Example 4 with TailLOUDSTrie

use of org.trie4j.louds.TailLOUDSTrie in project trie4j by takawitter.

the class CreateTail method main.

public static void main(String[] args) throws Exception {
    TailPatriciaTrie trie = new TailPatriciaTrie();
    for (String s : new WikipediaTitles("data/jawiki-20120220-all-titles-in-ns0.gz")) {
        trie.insert(s);
    }
    ConcatTailArrayBuilder ta = new ConcatTailArrayBuilder(trie.size());
    new TailLOUDSTrie(trie, ta);
    OutputStream os = new FileOutputStream("data/jawiki-20120220-tail");
    try {
    /*			CharSequence seq = ta.build().getTails();
			byte[] bytes = seq.toString().getBytes("UTF16");
			System.out.println(seq.length() + "chars.");
			System.out.println(bytes.length + "bytes.");
			os.write(bytes);
*/
    } finally {
        os.close();
    }
}