Examples with PinotDataBuffer com.linkedin.pinot.core.segment.memory.PinotDataBuffer used on opensource projects

Example 11 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedBitWidthRowColDataFileReaderTest method testSingleColUnsigned.

/**
   * Tests only positive numbers
   *
   * @throws Exception
   */
@Test
public void testSingleColUnsigned() throws Exception {
    int[] maxBitArray = new int[] { 4 };
    for (int maxBits : maxBitArray) {
        String fileName = "test" + maxBits + "FixedBitWidthSingleCol";
        File file = new File(fileName);
        CustomBitSet bitset = null;
        try {
            //        System.out.println("START MAX BITS:" + maxBits);
            int numElements = 100;
            bitset = CustomBitSet.withBitLength(numElements * maxBits);
            int max = (int) Math.pow(2, maxBits);
            Random r = new Random();
            int[] values = new int[numElements];
            for (int i = 0; i < numElements; i++) {
                int value = r.nextInt(max);
                values[i] = value;
                for (int j = maxBits - 1; j >= 0; j--) {
                    if ((value & (1 << j)) != 0) {
                        bitset.setBit(i * maxBits + (maxBits - j - 1));
                    }
                }
            }
            byte[] byteArray = bitset.toByteArray();
            FileOutputStream fos = new FileOutputStream(file);
            fos.write(byteArray);
            fos.close();
            PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
            FixedBitSingleValueMultiColReader heapReader = new FixedBitSingleValueMultiColReader(heapBuffer, numElements, 1, new int[] { maxBits });
            for (int i = 0; i < numElements; i++) {
                int readInt = heapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            heapReader.close();
            heapBuffer.close();
            PinotDataBuffer dataBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "mmap-testing");
            FixedBitSingleValueMultiColReader mmapReader = new FixedBitSingleValueMultiColReader(dataBuffer, numElements, 1, new int[] { maxBits });
            for (int i = 0; i < numElements; i++) {
                int readInt = mmapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            mmapReader.close();
            dataBuffer.close();
        //        System.out.println("END MAX BITS:" + maxBits);
        } finally {
            file.delete();
            bitset.close();
        }
    }
}

Example 12 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class ColumnIndexDirectoryTestHelper method newIndexBuffer.

static PinotDataBuffer newIndexBuffer(ColumnIndexDirectory columnDirectory, String column, int size, int index) throws IOException {
    String columnName = column + "." + Integer.toString(index);
    // skip star tree. It's managed differently
    ColumnIndexType indexType = indexTypes[index % indexTypes.length];
    PinotDataBuffer buf = null;
    switch(indexType) {
        case DICTIONARY:
            buf = columnDirectory.newDictionaryBuffer(columnName, size);
            break;
        case FORWARD_INDEX:
            buf = columnDirectory.newForwardIndexBuffer(columnName, size);
            break;
        case INVERTED_INDEX:
            buf = columnDirectory.newInvertedIndexBuffer(columnName, size);
            break;
    }
    return buf;
}

Example 13 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class ColumnIndexDirectoryTestHelper method getIndexBuffer.

static PinotDataBuffer getIndexBuffer(ColumnIndexDirectory columnDirectory, String column, int index) throws IOException {
    String columnName = column + "." + Integer.toString(index);
    // skip star tree
    ColumnIndexType indexType = indexTypes[index % indexTypes.length];
    PinotDataBuffer buf = null;
    switch(indexType) {
        case DICTIONARY:
            buf = columnDirectory.getDictionaryBufferFor(columnName);
            break;
        case FORWARD_INDEX:
            buf = columnDirectory.getForwardIndexBufferFor(columnName);
            break;
        case INVERTED_INDEX:
            buf = columnDirectory.getInvertedIndexBufferFor(columnName);
            break;
    }
    return buf;
}

Example 14 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FilePerIndexDirectoryTest method testHasIndex.

@Test
public void testHasIndex() throws IOException {
    try (FilePerIndexDirectory fpiDirectory = new FilePerIndexDirectory(segmentDir, segmentMetadata, ReadMode.mmap)) {
        PinotDataBuffer buffer = fpiDirectory.newDictionaryBuffer("foo", 1024);
        buffer.putInt(0, 100);
        Assert.assertTrue(fpiDirectory.hasIndexFor("foo", ColumnIndexType.DICTIONARY));
    }
}

Example 15 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class SingleFileIndexDirectoryTest method testRemoveIndex.

@Test(expectedExceptions = UnsupportedOperationException.class)
public void testRemoveIndex() throws IOException, ConfigurationException {
    try (SingleFileIndexDirectory sfd = new SingleFileIndexDirectory(segmentDir, segmentMetadata, ReadMode.mmap)) {
        try (PinotDataBuffer buffer = sfd.newDictionaryBuffer("col1", 1024)) {
        }
        Assert.assertFalse(sfd.isIndexRemovalSupported());
        sfd.removeIndex("col1", ColumnIndexType.DICTIONARY);
    }
}