Examples with SortedForwardIndexReader com.linkedin.pinot.core.io.reader.impl.SortedForwardIndexReader used on opensource projects

Example 1 with SortedForwardIndexReader

use of com.linkedin.pinot.core.io.reader.impl.SortedForwardIndexReader in project pinot by linkedin.

the class SortedForwardIndexReaderTest method testSimple.

public void testSimple() throws Exception {
    int maxLength = 1000;
    int cardinality = 100000;
    File file = new File("test_sortef_fwd_index.dat");
    file.delete();
    int[] columnSizes = new int[] { 4, 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(file, cardinality, columnSizes.length, columnSizes);
    Random random = new Random();
    int[] startDocIdArray = new int[cardinality];
    int[] endDocIdArray = new int[cardinality];
    int prevEnd = -1;
    int totalDocs = 0;
    for (int i = 0; i < cardinality; i++) {
        int length = random.nextInt(maxLength);
        int start = prevEnd + 1;
        int end = start + length;
        startDocIdArray[i] = start;
        endDocIdArray[i] = end;
        writer.setInt(i, 0, start);
        writer.setInt(i, 1, end);
        prevEnd = end;
        totalDocs += length;
    }
    writer.close();
    PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
    FixedByteSingleValueMultiColReader rawFileReader = new FixedByteSingleValueMultiColReader(heapBuffer, cardinality, columnSizes.length, columnSizes);
    SortedForwardIndexReader reader = new SortedForwardIndexReader(rawFileReader, totalDocs);
    // without using context
    long start, end;
    start = System.currentTimeMillis();
    for (int i = 0; i < cardinality; i++) {
        for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
            Assert.assertEquals(reader.getInt(docId), i);
        }
    }
    end = System.currentTimeMillis();
    System.out.println("Took " + (end - start) + " to scan " + totalDocs + " docs without using context");
    // with context
    SortedValueReaderContext context = reader.createContext();
    start = System.currentTimeMillis();
    for (int i = 0; i < cardinality; i++) {
        for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
            Assert.assertEquals(reader.getInt(docId, context), i);
        }
    }
    end = System.currentTimeMillis();
    LOGGER.debug("Took " + (end - start) + " to scan " + totalDocs + " with context");
    reader.close();
    file.delete();
    heapBuffer.close();
}

Example 2 with SortedForwardIndexReader

use of com.linkedin.pinot.core.io.reader.impl.SortedForwardIndexReader in project pinot by linkedin.

the class ColumnDataSourceImpl method getNextBlock.

@Override
public Block getNextBlock(BlockId blockId) {
    Block b = null;
    ColumnMetadata columnMetadata = indexContainer.getColumnMetadata();
    if (columnMetadata.isSingleValue()) {
        // TODO: Support sorted index without dictionary.
        if (columnMetadata.hasDictionary() && columnMetadata.isSorted()) {
            b = new SortedSingleValueBlock(blockId, (SortedForwardIndexReader) indexContainer.getForwardIndex(), indexContainer.getDictionary(), columnMetadata);
        } else {
            b = new UnSortedSingleValueBlock(blockId, (SingleColumnSingleValueReader) indexContainer.getForwardIndex(), indexContainer.getDictionary(), columnMetadata);
        }
    } else {
        b = new MultiValueBlock(blockId, (SingleColumnMultiValueReader) indexContainer.getForwardIndex(), indexContainer.getDictionary(), columnMetadata);
    }
    return b;
}