Examples with RealtimeInvertedIndex com.linkedin.pinot.core.realtime.impl.invertedIndex.RealtimeInvertedIndex used on opensource projects

Example 1 with RealtimeInvertedIndex

use of com.linkedin.pinot.core.realtime.impl.invertedIndex.RealtimeInvertedIndex in project pinot by linkedin.

the class RealtimeSegmentImpl method destroy.

@Override
public void destroy() {
    LOGGER.info("Trying to close RealtimeSegmentImpl : {}", this.getSegmentName());
    for (DataFileReader dfReader : columnIndexReaderWriterMap.values()) {
        try {
            dfReader.close();
        } catch (IOException e) {
            LOGGER.error("Failed to close index. Service will continue with potential memory leak, error: ", e);
        // fall through to close other segments
        }
    }
    // clear map now that index is closed to prevent accidental usage
    columnIndexReaderWriterMap.clear();
    for (RealtimeInvertedIndex index : invertedIndexMap.values()) {
        try {
            index.close();
        } catch (IOException e) {
            LOGGER.error("Failed to close inverted index. Service will continue with memory leaks, error: ", e);
        }
    }
    invertedIndexMap.clear();
    _segmentMetadata.close();
}

Example 2 with RealtimeInvertedIndex

use of com.linkedin.pinot.core.realtime.impl.invertedIndex.RealtimeInvertedIndex in project pinot by linkedin.

the class RealtimeSegmentImpl method getSortedBitmapIntIteratorsForLongColumn.

private IntIterator[] getSortedBitmapIntIteratorsForLongColumn(final String columnToSortOn) {
    final RealtimeInvertedIndex index = invertedIndexMap.get(columnToSortOn);
    final MutableDictionaryReader dictionary = dictionaryMap.get(columnToSortOn);
    final IntIterator[] intIterators = new IntIterator[dictionary.length()];
    final List<Long> rawValues = new ArrayList<Long>();
    for (int i = 0; i < dictionary.length(); i++) {
        rawValues.add((Long) dictionary.get(i));
    }
    long start = System.currentTimeMillis();
    Collections.sort(rawValues);
    LOGGER.info("Column {}, dictionary len : {}, time to sort : {} ", columnToSortOn, dictionary.length(), (System.currentTimeMillis() - start));
    for (int i = 0; i < rawValues.size(); i++) {
        intIterators[i] = index.getDocIdSetFor(dictionary.indexOf(rawValues.get(i))).getIntIterator();
    }
    return intIterators;
}

Example 3 with RealtimeInvertedIndex

use of com.linkedin.pinot.core.realtime.impl.invertedIndex.RealtimeInvertedIndex in project pinot by linkedin.

the class RealtimeSegmentImpl method getSortedBitmapIntIteratorsForDoubleColumn.

private IntIterator[] getSortedBitmapIntIteratorsForDoubleColumn(final String columnToSortOn) {
    final RealtimeInvertedIndex index = invertedIndexMap.get(columnToSortOn);
    final MutableDictionaryReader dictionary = dictionaryMap.get(columnToSortOn);
    final IntIterator[] intIterators = new IntIterator[dictionary.length()];
    final List<Double> rawValues = new ArrayList<Double>();
    for (int i = 0; i < dictionary.length(); i++) {
        rawValues.add((Double) dictionary.get(i));
    }
    long start = System.currentTimeMillis();
    Collections.sort(rawValues);
    LOGGER.info("Column {}, dictionary len : {}, time to sort : {} ", columnToSortOn, dictionary.length(), (System.currentTimeMillis() - start));
    for (int i = 0; i < rawValues.size(); i++) {
        intIterators[i] = index.getDocIdSetFor(dictionary.indexOf(rawValues.get(i))).getIntIterator();
    }
    return intIterators;
}

Example 4 with RealtimeInvertedIndex

use of com.linkedin.pinot.core.realtime.impl.invertedIndex.RealtimeInvertedIndex in project pinot by linkedin.

the class RealtimeSegmentImpl method getSortedBitmapIntIteratorsForFloatColumn.

private IntIterator[] getSortedBitmapIntIteratorsForFloatColumn(final String columnToSortOn) {
    final RealtimeInvertedIndex index = invertedIndexMap.get(columnToSortOn);
    final MutableDictionaryReader dictionary = dictionaryMap.get(columnToSortOn);
    final IntIterator[] intIterators = new IntIterator[dictionary.length()];
    final List<Float> rawValues = new ArrayList<Float>();
    for (int i = 0; i < dictionary.length(); i++) {
        rawValues.add((Float) dictionary.get(i));
    }
    long start = System.currentTimeMillis();
    Collections.sort(rawValues);
    LOGGER.info("Column {}, dictionary len : {}, time to sort : {} ", columnToSortOn, dictionary.length(), (System.currentTimeMillis() - start));
    for (int i = 0; i < rawValues.size(); i++) {
        intIterators[i] = index.getDocIdSetFor(dictionary.indexOf(rawValues.get(i))).getIntIterator();
    }
    return intIterators;
}

Example 5 with RealtimeInvertedIndex

use of com.linkedin.pinot.core.realtime.impl.invertedIndex.RealtimeInvertedIndex in project pinot by linkedin.

the class RealtimeSegmentImpl method getSortedBitmapIntIteratorsForStringColumn.

private IntIterator[] getSortedBitmapIntIteratorsForStringColumn(final String columnToSortOn) {
    final RealtimeInvertedIndex index = invertedIndexMap.get(columnToSortOn);
    final MutableDictionaryReader dictionary = dictionaryMap.get(columnToSortOn);
    final IntIterator[] intIterators = new IntIterator[dictionary.length()];
    final List<String> rawValues = new ArrayList<String>();
    for (int i = 0; i < dictionary.length(); i++) {
        rawValues.add((String) dictionary.get(i));
    }
    long start = System.currentTimeMillis();
    Collections.sort(rawValues);
    LOGGER.info("Column {}, dictionary len : {}, time to sort : {} ", columnToSortOn, dictionary.length(), (System.currentTimeMillis() - start));
    for (int i = 0; i < rawValues.size(); i++) {
        intIterators[i] = index.getDocIdSetFor(dictionary.indexOf(rawValues.get(i))).getIntIterator();
    }
    return intIterators;
}