Examples with InMemoryLookupCache org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache used on opensource projects

Example 6 with InMemoryLookupCache

use of org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache in project deeplearning4j by deeplearning4j.

the class VocabularyHolderTest method testTransferBackToVocabCache.

@Test
public void testTransferBackToVocabCache() throws Exception {
    VocabularyHolder holder = new VocabularyHolder();
    holder.addWord("test");
    holder.addWord("tests");
    holder.addWord("testz");
    holder.incrementWordCounter("tests");
    holder.incrementWordCounter("tests");
    holder.incrementWordCounter("testz");
    InMemoryLookupCache cache = new InMemoryLookupCache(false);
    holder.updateHuffmanCodes();
    holder.transferBackToVocabCache(cache);
    // checking word frequency transfer
    assertEquals(3, cache.numWords());
    assertEquals(1, cache.wordFrequency("test"));
    assertEquals(2, cache.wordFrequency("testz"));
    assertEquals(3, cache.wordFrequency("tests"));
    // checking Huffman tree transfer
    assertEquals("tests", cache.wordAtIndex(0));
    assertEquals("testz", cache.wordAtIndex(1));
    assertEquals("test", cache.wordAtIndex(2));
}

Example 7 with InMemoryLookupCache

use of org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache in project deeplearning4j by deeplearning4j.

the class WordVectorSerializerTest method testWriteWordVectors.

@Test
@Ignore
public void testWriteWordVectors() throws IOException {
    WordVectors vec = WordVectorSerializer.loadGoogleModel(binaryFile, true);
    InMemoryLookupTable lookupTable = (InMemoryLookupTable) vec.lookupTable();
    InMemoryLookupCache lookupCache = (InMemoryLookupCache) vec.vocab();
    WordVectorSerializer.writeWordVectors(lookupTable, lookupCache, pathToWriteto);
    WordVectors wordVectors = WordVectorSerializer.loadTxtVectors(new File(pathToWriteto));
    double[] wordVector1 = wordVectors.getWordVector("Morgan_Freeman");
    double[] wordVector2 = wordVectors.getWordVector("JA_Montalbano");
    assertTrue(wordVector1.length == 300);
    assertTrue(wordVector2.length == 300);
    assertEquals(Doubles.asList(wordVector1).get(0), 0.044423, 1e-3);
    assertEquals(Doubles.asList(wordVector2).get(0), 0.051964, 1e-3);
}

Example 8 with InMemoryLookupCache

use of org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache in project deeplearning4j by deeplearning4j.

the class WordVectorSerializerTest method testFullModelSerialization.

@Test
public void testFullModelSerialization() throws Exception {
    File inputFile = new ClassPathResource("/big/raw_sentences.txt").getFile();
    SentenceIterator iter = UimaSentenceIterator.createWithPath(inputFile.getAbsolutePath());
    // Split on white spaces in the line to get words
    TokenizerFactory t = new DefaultTokenizerFactory();
    t.setTokenPreProcessor(new CommonPreprocessor());
    InMemoryLookupCache cache = new InMemoryLookupCache(false);
    WeightLookupTable table = new InMemoryLookupTable.Builder().vectorLength(100).useAdaGrad(false).negative(5.0).cache(cache).lr(0.025f).build();
    Word2Vec vec = new Word2Vec.Builder().minWordFrequency(5).iterations(1).epochs(1).layerSize(100).lookupTable(table).stopWords(new ArrayList<String>()).useAdaGrad(false).negativeSample(5).vocabCache(cache).seed(42).windowSize(5).iterate(iter).tokenizerFactory(t).build();
    assertEquals(new ArrayList<String>(), vec.getStopWords());
    vec.fit();
    //logger.info("Original word 0: " + cache.wordFor(cache.wordAtIndex(0)));
    //logger.info("Closest Words:");
    Collection<String> lst = vec.wordsNearest("day", 10);
    System.out.println(lst);
    WordVectorSerializer.writeFullModel(vec, "tempModel.txt");
    File modelFile = new File("tempModel.txt");
    modelFile.deleteOnExit();
    assertTrue(modelFile.exists());
    assertTrue(modelFile.length() > 0);
    Word2Vec vec2 = WordVectorSerializer.loadFullModel("tempModel.txt");
    assertNotEquals(null, vec2);
    assertEquals(vec.getConfiguration(), vec2.getConfiguration());
    //logger.info("Source ExpTable: " + ArrayUtils.toString(((InMemoryLookupTable) table).getExpTable()));
    //logger.info("Dest  ExpTable: " + ArrayUtils.toString(((InMemoryLookupTable)  vec2.getLookupTable()).getExpTable()));
    assertTrue(ArrayUtils.isEquals(((InMemoryLookupTable) table).getExpTable(), ((InMemoryLookupTable) vec2.getLookupTable()).getExpTable()));
    InMemoryLookupTable restoredTable = (InMemoryLookupTable) vec2.lookupTable();
    /*
        logger.info("Restored word 1: " + restoredTable.getVocab().wordFor(restoredTable.getVocab().wordAtIndex(1)));
        logger.info("Restored word 'it': " + restoredTable.getVocab().wordFor("it"));
        logger.info("Original word 1: " + cache.wordFor(cache.wordAtIndex(1)));
        logger.info("Original word 'i': " + cache.wordFor("i"));
        logger.info("Original word 0: " + cache.wordFor(cache.wordAtIndex(0)));
        logger.info("Restored word 0: " + restoredTable.getVocab().wordFor(restoredTable.getVocab().wordAtIndex(0)));
        */
    assertEquals(cache.wordAtIndex(1), restoredTable.getVocab().wordAtIndex(1));
    assertEquals(cache.wordAtIndex(7), restoredTable.getVocab().wordAtIndex(7));
    assertEquals(cache.wordAtIndex(15), restoredTable.getVocab().wordAtIndex(15));
    /*
            these tests needed only to make sure INDArray equality is working properly
         */
    double[] array1 = new double[] { 0.323232325, 0.65756575, 0.12315, 0.12312315, 0.1232135, 0.12312315, 0.4343423425, 0.15 };
    double[] array2 = new double[] { 0.423232325, 0.25756575, 0.12375, 0.12311315, 0.1232035, 0.12318315, 0.4343493425, 0.25 };
    assertNotEquals(Nd4j.create(array1), Nd4j.create(array2));
    assertEquals(Nd4j.create(array1), Nd4j.create(array1));
    INDArray rSyn0_1 = restoredTable.getSyn0().slice(1);
    INDArray oSyn0_1 = ((InMemoryLookupTable) table).getSyn0().slice(1);
    //logger.info("Restored syn0: " + rSyn0_1);
    //logger.info("Original syn0: " + oSyn0_1);
    assertEquals(oSyn0_1, rSyn0_1);
    // just checking $^###! syn0/syn1 order
    int cnt = 0;
    for (VocabWord word : cache.vocabWords()) {
        INDArray rSyn0 = restoredTable.getSyn0().slice(word.getIndex());
        INDArray oSyn0 = ((InMemoryLookupTable) table).getSyn0().slice(word.getIndex());
        assertEquals(rSyn0, oSyn0);
        assertEquals(1.0, arraysSimilarity(rSyn0, oSyn0), 0.001);
        INDArray rSyn1 = restoredTable.getSyn1().slice(word.getIndex());
        INDArray oSyn1 = ((InMemoryLookupTable) table).getSyn1().slice(word.getIndex());
        assertEquals(rSyn1, oSyn1);
        if (arraysSimilarity(rSyn1, oSyn1) < 0.98) {
        //   logger.info("Restored syn1: " + rSyn1);
        //   logger.info("Original  syn1: " + oSyn1);
        }
        // we exclude word 222 since it has syn1 full of zeroes
        if (cnt != 222)
            assertEquals(1.0, arraysSimilarity(rSyn1, oSyn1), 0.001);
        if (((InMemoryLookupTable) table).getSyn1Neg() != null) {
            INDArray rSyn1Neg = restoredTable.getSyn1Neg().slice(word.getIndex());
            INDArray oSyn1Neg = ((InMemoryLookupTable) table).getSyn1Neg().slice(word.getIndex());
            assertEquals(rSyn1Neg, oSyn1Neg);
        //                assertEquals(1.0, arraysSimilarity(rSyn1Neg, oSyn1Neg), 0.001);
        }
        assertEquals(word.getHistoricalGradient(), restoredTable.getVocab().wordFor(word.getWord()).getHistoricalGradient());
        cnt++;
    }
    // at this moment we can assume that whole model is transferred, and we can call fit over new model
    //        iter.reset();
    iter = UimaSentenceIterator.createWithPath(inputFile.getAbsolutePath());
    vec2.setTokenizerFactory(t);
    vec2.setSentenceIterator(iter);
    vec2.fit();
    INDArray day1 = vec.getWordVectorMatrix("day");
    INDArray day2 = vec2.getWordVectorMatrix("day");
    INDArray night1 = vec.getWordVectorMatrix("night");
    INDArray night2 = vec2.getWordVectorMatrix("night");
    double simD = arraysSimilarity(day1, day2);
    double simN = arraysSimilarity(night1, night2);
    logger.info("Vec1 day: " + day1);
    logger.info("Vec2 day: " + day2);
    logger.info("Vec1 night: " + night1);
    logger.info("Vec2 night: " + night2);
    logger.info("Day/day cross-model similarity: " + simD);
    logger.info("Night/night cross-model similarity: " + simN);
    logger.info("Vec1 day/night similiraty: " + vec.similarity("day", "night"));
    logger.info("Vec2 day/night similiraty: " + vec2.similarity("day", "night"));
    // check if cross-model values are not the same
    assertNotEquals(1.0, simD, 0.001);
    assertNotEquals(1.0, simN, 0.001);
    // check if cross-model values are still close to each other
    assertTrue(simD > 0.70);
    assertTrue(simN > 0.70);
    modelFile.delete();
}

Example 9 with InMemoryLookupCache

use of org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache in project deeplearning4j by deeplearning4j.

the class WordVectorSerializerTest method testOutputStream.

@Test
public void testOutputStream() throws Exception {
    File file = File.createTempFile("tmp_ser", "ssa");
    file.deleteOnExit();
    File inputFile = new ClassPathResource("/big/raw_sentences.txt").getFile();
    SentenceIterator iter = new BasicLineIterator(inputFile);
    // Split on white spaces in the line to get words
    TokenizerFactory t = new DefaultTokenizerFactory();
    t.setTokenPreProcessor(new CommonPreprocessor());
    InMemoryLookupCache cache = new InMemoryLookupCache(false);
    WeightLookupTable table = new InMemoryLookupTable.Builder().vectorLength(100).useAdaGrad(false).negative(5.0).cache(cache).lr(0.025f).build();
    Word2Vec vec = new Word2Vec.Builder().minWordFrequency(5).iterations(1).epochs(1).layerSize(100).lookupTable(table).stopWords(new ArrayList<String>()).useAdaGrad(false).negativeSample(5).vocabCache(cache).seed(42).windowSize(5).iterate(iter).tokenizerFactory(t).build();
    assertEquals(new ArrayList<String>(), vec.getStopWords());
    vec.fit();
    INDArray day1 = vec.getWordVectorMatrix("day");
    WordVectorSerializer.writeWordVectors(vec, new FileOutputStream(file));
    WordVectors vec2 = WordVectorSerializer.loadTxtVectors(file);
    INDArray day2 = vec2.getWordVectorMatrix("day");
    assertEquals(day1, day2);
    File tempFile = File.createTempFile("tetsts", "Fdfs");
    tempFile.deleteOnExit();
    WordVectorSerializer.writeWord2VecModel(vec, tempFile);
    Word2Vec vec3 = WordVectorSerializer.readWord2VecModel(tempFile);
}

Example 10 with InMemoryLookupCache

use of org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache in project deeplearning4j by deeplearning4j.

the class VocabularyHolder method transferBackToVocabCache.

/**
     * This method is required for compatibility purposes.
     *  It just transfers vocabulary from VocabHolder into VocabCache
     *
     * @param cache
     */
public void transferBackToVocabCache(VocabCache cache, boolean emptyHolder) {
    if (!(cache instanceof InMemoryLookupCache))
        throw new IllegalStateException("Sorry, only InMemoryLookupCache use implemented.");
    // make sure that huffman codes are updated before transfer
    //updateHuffmanCodes();
    List<VocabularyWord> words = words();
    for (VocabularyWord word : words) {
        if (word.getWord().isEmpty())
            continue;
        VocabWord vocabWord = new VocabWord(1, word.getWord());
        // if we're transferring full model, it CAN contain HistoricalGradient for AdaptiveGradient feature
        if (word.getHistoricalGradient() != null) {
            INDArray gradient = Nd4j.create(word.getHistoricalGradient());
            vocabWord.setHistoricalGradient(gradient);
        }
        // put VocabWord into both Tokens and Vocabs maps
        ((InMemoryLookupCache) cache).getVocabs().put(word.getWord(), vocabWord);
        ((InMemoryLookupCache) cache).getTokens().put(word.getWord(), vocabWord);
        // update Huffman tree information
        if (word.getHuffmanNode() != null) {
            vocabWord.setIndex(word.getHuffmanNode().getIdx());
            vocabWord.setCodeLength(word.getHuffmanNode().getLength());
            vocabWord.setPoints(arrayToList(word.getHuffmanNode().getPoint(), word.getHuffmanNode().getLength()));
            vocabWord.setCodes(arrayToList(word.getHuffmanNode().getCode(), word.getHuffmanNode().getLength()));
            // put word into index
            cache.addWordToIndex(word.getHuffmanNode().getIdx(), word.getWord());
        }
        // >1 hack is required since VocabCache impl imples 1 as base word count, not 0
        if (word.getCount() > 1)
            cache.incrementWordCount(word.getWord(), word.getCount() - 1);
    }
    // at this moment its pretty safe to nullify all vocabs.
    if (emptyHolder) {
        idxMap.clear();
        vocabulary.clear();
    }
}