Examples with ReaderContext com.linkedin.pinot.core.io.reader.ReaderContext used on opensource projects

Example 1 with ReaderContext

use of com.linkedin.pinot.core.io.reader.ReaderContext in project pinot by linkedin.

the class UnSortedSingleValueSet method getDoubleValues.

/**
   * Reads double values for the given docIds and returns in the passed in double[].
   * Compatible types (int, float, long) can also be read in as double.
   *
   * @param inDocIds DocIds for which to get the values
   * @param inStartPos start index in the inDocIds array
   * @param inDocIdsSize size of docIds to read
   * @param outValues Array where output is written
   * @param outStartPos start index into output array
   */
@Override
public void getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos) {
    int inEndPos = inStartPos + inDocIdsSize;
    ReaderContext context = sVReader.createContext();
    switch(columnMetadata.getDataType()) {
        case INT:
            for (int i = inStartPos; i < inEndPos; i++) {
                outValues[outStartPos++] = sVReader.getInt(inDocIds[i], context);
            }
            break;
        case LONG:
            for (int i = inStartPos; i < inEndPos; i++) {
                outValues[outStartPos++] = sVReader.getLong(inDocIds[i], context);
            }
            break;
        case FLOAT:
            for (int i = inStartPos; i < inEndPos; i++) {
                outValues[outStartPos++] = sVReader.getFloat(inDocIds[i], context);
            }
            break;
        case DOUBLE:
            for (int i = inStartPos; i < inEndPos; i++) {
                outValues[outStartPos++] = sVReader.getDouble(inDocIds[i], context);
            }
            break;
        default:
            throw new UnsupportedOperationException("Cannot fetch double values for column: " + columnMetadata.getColumnName());
    }
}

Example 2 with ReaderContext

use of com.linkedin.pinot.core.io.reader.ReaderContext in project pinot by linkedin.

the class UnSortedSingleValueSet method getStringValues.

@Override
public void getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos) {
    ReaderContext context = sVReader.createContext();
    int inEndPos = inStartPos + inDocIdsSize;
    try {
        for (int i = inStartPos; i < inEndPos; i++) {
            outValues[outStartPos++] = sVReader.getString(inDocIds[i], context);
        }
    } catch (Exception e) {
        Utils.rethrowException(e);
    }
}

Example 3 with ReaderContext

use of com.linkedin.pinot.core.io.reader.ReaderContext in project pinot by linkedin.

the class FixedBitMultiValueTest method testSingleColMultiValueWithContext.

public void testSingleColMultiValueWithContext(Class<? extends SingleColumnMultiValueWriter> writerClazz, Class<? extends SingleColumnMultiValueReader<? extends ReaderContext>> readerClazz) throws Exception {
    LOGGER.info("Testing for writerClazz:{} readerClass:{}", writerClazz.getName(), readerClazz.getName());
    Constructor<? extends SingleColumnMultiValueWriter> writerClazzConstructor = writerClazz.getConstructor(new Class[] { File.class, int.class, int.class, int.class });
    Constructor<? extends SingleColumnMultiValueReader<? extends ReaderContext>> readerClazzConstructor = readerClazz.getConstructor(new Class[] { PinotDataBuffer.class, int.class, int.class, int.class, boolean.class });
    int maxBits = 1;
    while (maxBits < 32) {
        final String fileName = getClass().getName() + "_test_single_col_mv_fixed_bit.dat";
        final File f = new File(fileName);
        f.delete();
        int numDocs = 10;
        int maxNumValues = 100;
        final int[][] data = new int[numDocs][];
        final Random r = new Random();
        final int maxValue = (int) Math.pow(2, maxBits);
        int totalNumValues = 0;
        int[] startOffsets = new int[numDocs];
        int[] lengths = new int[numDocs];
        for (int i = 0; i < data.length; i++) {
            final int numValues = r.nextInt(maxNumValues) + 1;
            data[i] = new int[numValues];
            for (int j = 0; j < numValues; j++) {
                data[i][j] = r.nextInt(maxValue);
            }
            startOffsets[i] = totalNumValues;
            lengths[i] = numValues;
            totalNumValues = totalNumValues + numValues;
        }
        SingleColumnMultiValueWriter writer = writerClazzConstructor.newInstance(new Object[] { f, numDocs, totalNumValues, maxBits });
        for (int i = 0; i < data.length; i++) {
            writer.setIntArray(i, data[i]);
        }
        writer.close();
        final RandomAccessFile raf = new RandomAccessFile(f, "rw");
        raf.close();
        // Test heap mode
        PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(f, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
        SingleColumnMultiValueReader heapReader = readerClazzConstructor.newInstance(new Object[] { heapBuffer, numDocs, totalNumValues, maxBits, false });
        ReaderContext context = heapReader.createContext();
        final int[] readValues = new int[maxNumValues];
        for (int i = 0; i < data.length; i++) {
            final int numValues = heapReader.getIntArray(i, readValues, context);
            if (numValues != data[i].length) {
                System.err.println("Failed Expected:" + data[i].length + " Actual:" + numValues);
                int length = heapReader.getIntArray(i, readValues, context);
            }
            Assert.assertEquals(numValues, data[i].length);
            for (int j = 0; j < numValues; j++) {
                Assert.assertEquals(readValues[j], data[i][j]);
            }
        }
        heapReader.close();
        heapBuffer.close();
        // Test mmap mode
        PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(f, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");
        SingleColumnMultiValueReader<? extends ReaderContext> mmapReader = readerClazzConstructor.newInstance(new Object[] { mmapBuffer, numDocs, totalNumValues, maxBits, false });
        for (int i = 0; i < data.length; i++) {
            final int numValues = mmapReader.getIntArray(i, readValues);
            Assert.assertEquals(numValues, data[i].length);
            for (int j = 0; j < numValues; j++) {
                Assert.assertEquals(readValues[j], data[i][j]);
            }
        }
        mmapReader.close();
        mmapBuffer.close();
        f.delete();
        maxBits = maxBits + 1;
    }
    LOGGER.info("DONE: Testing for writerClazz:{} readerClass:{}", writerClazz.getName(), readerClazz.getName());
}

Example 4 with ReaderContext

use of com.linkedin.pinot.core.io.reader.ReaderContext in project pinot by linkedin.

the class UnSortedSingleValueSet method getFloatValues.

/**
   * Reads int values for the given docIds and returns in the passed in double[].
   * Compatible data types ('int', 'long' and 'float') can be read in as float.
   *
   * @param inDocIds DocIds for which to get the values
   * @param inStartPos start index in the inDocIds array
   * @param inDocIdsSize size of docIds to read
   * @param outValues Array where output is written
   * @param outStartPos start index into output array
   */
@Override
public void getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos) {
    int inEndPos = inStartPos + inDocIdsSize;
    ReaderContext context = sVReader.createContext();
    switch(columnMetadata.getDataType()) {
        case INT:
            for (int i = inStartPos; i < inEndPos; i++) {
                outValues[outStartPos++] = sVReader.getInt(inDocIds[i], context);
            }
            break;
        case LONG:
            for (int i = inStartPos; i < inEndPos; i++) {
                outValues[outStartPos++] = sVReader.getLong(inDocIds[i], context);
            }
            break;
        case FLOAT:
            for (int i = inStartPos; i < inEndPos; i++) {
                outValues[outStartPos++] = sVReader.getFloat(inDocIds[i], context);
            }
            break;
        default:
            throw new UnsupportedOperationException("Cannot fetch float values for column: " + columnMetadata.getColumnName());
    }
}

Example 5 with ReaderContext

use of com.linkedin.pinot.core.io.reader.ReaderContext in project pinot by linkedin.

the class UnSortedSingleValueSet method getIntValues.

/**
   * Reads int values for the given docIds and returns in the passed in double[].
   * Only 'int' data type allowed to be read in as 'int'.
   *
   * @param inDocIds DocIds for which to get the values
   * @param inStartPos start index in the inDocIds array
   * @param inDocIdsSize size of docIds to read
   * @param outValues Array where output is written
   * @param outStartPos start index into output array
   */
@Override
public void getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos) {
    int inEndPos = inStartPos + inDocIdsSize;
    ReaderContext context = sVReader.createContext();
    if (columnMetadata.getDataType() == DataType.INT) {
        for (int i = inStartPos; i < inEndPos; i++) {
            outValues[outStartPos++] = sVReader.getInt(inDocIds[i], context);
        }
    } else {
        throw new UnsupportedOperationException("Cannot fetch int values for column: " + columnMetadata.getColumnName());
    }
}