Examples with SortedValueReaderContext com.linkedin.pinot.core.io.reader.impl.SortedValueReaderContext used on opensource projects

Example 1 with SortedValueReaderContext

use of com.linkedin.pinot.core.io.reader.impl.SortedValueReaderContext in project pinot by linkedin.

the class SortedSingleValueSet method getDictionaryIds.

@Override
public void getDictionaryIds(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outDictionaryIds, int outStartPos) {
    SortedValueReaderContext readerContext = sVReader.createContext();
    int endPos = inStartPos + inDocIdsSize;
    for (int iter = inStartPos; iter < endPos; iter++) {
        int row = inDocIds[iter];
        outDictionaryIds[outStartPos++] = sVReader.getInt(row, readerContext);
    }
}

Example 2 with SortedValueReaderContext

use of com.linkedin.pinot.core.io.reader.impl.SortedValueReaderContext in project pinot by linkedin.

the class SortedForwardIndexReaderTest method testSimple.

public void testSimple() throws Exception {
    int maxLength = 1000;
    int cardinality = 100000;
    File file = new File("test_sortef_fwd_index.dat");
    file.delete();
    int[] columnSizes = new int[] { 4, 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(file, cardinality, columnSizes.length, columnSizes);
    Random random = new Random();
    int[] startDocIdArray = new int[cardinality];
    int[] endDocIdArray = new int[cardinality];
    int prevEnd = -1;
    int totalDocs = 0;
    for (int i = 0; i < cardinality; i++) {
        int length = random.nextInt(maxLength);
        int start = prevEnd + 1;
        int end = start + length;
        startDocIdArray[i] = start;
        endDocIdArray[i] = end;
        writer.setInt(i, 0, start);
        writer.setInt(i, 1, end);
        prevEnd = end;
        totalDocs += length;
    }
    writer.close();
    PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
    FixedByteSingleValueMultiColReader rawFileReader = new FixedByteSingleValueMultiColReader(heapBuffer, cardinality, columnSizes.length, columnSizes);
    SortedForwardIndexReader reader = new SortedForwardIndexReader(rawFileReader, totalDocs);
    // without using context
    long start, end;
    start = System.currentTimeMillis();
    for (int i = 0; i < cardinality; i++) {
        for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
            Assert.assertEquals(reader.getInt(docId), i);
        }
    }
    end = System.currentTimeMillis();
    System.out.println("Took " + (end - start) + " to scan " + totalDocs + " docs without using context");
    // with context
    SortedValueReaderContext context = reader.createContext();
    start = System.currentTimeMillis();
    for (int i = 0; i < cardinality; i++) {
        for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
            Assert.assertEquals(reader.getInt(docId, context), i);
        }
    }
    end = System.currentTimeMillis();
    LOGGER.debug("Took " + (end - start) + " to scan " + totalDocs + " with context");
    reader.close();
    file.delete();
    heapBuffer.close();
}