Examples with SynchronizedSentenceIterator org.deeplearning4j.text.sentenceiterator.SynchronizedSentenceIterator used on opensource projects

Example 1 with SynchronizedSentenceIterator

use of org.deeplearning4j.text.sentenceiterator.SynchronizedSentenceIterator in project deeplearning4j by deeplearning4j.

the class AbstractCoOccurrences method iterator.

/**
     *
     *  This method returns iterator with elements pairs and their weights. Resulting iterator is safe to use in multi-threaded environment.
     *
     * Developer's note: thread safety on received iterator is delegated to PrefetchedSentenceIterator
     * @return
     */
public Iterator<Pair<Pair<T, T>, Double>> iterator() {
    final SentenceIterator iterator;
    try {
        iterator = new SynchronizedSentenceIterator(new PrefetchingSentenceIterator.Builder(new BasicLineIterator(targetFile)).setFetchSize(500000).build());
    } catch (Exception e) {
        logger.error("Target file was not found on last stage!");
        throw new RuntimeException(e);
    }
    return new Iterator<Pair<Pair<T, T>, Double>>() {

        /*
                    iterator should be built on top of current text file with all pairs
             */
        @Override
        public boolean hasNext() {
            return iterator.hasNext();
        }

        @Override
        public Pair<Pair<T, T>, Double> next() {
            String line = iterator.nextSentence();
            String[] strings = line.split(" ");
            T element1 = vocabCache.elementAtIndex(Integer.valueOf(strings[0]));
            T element2 = vocabCache.elementAtIndex(Integer.valueOf(strings[1]));
            Double weight = Double.valueOf(strings[2]);
            return new Pair<>(new Pair<>(element1, element2), weight);
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException("remove() method can't be supported on read-only interface");
        }
    };
}