Examples with MeasureRawColumnChunk org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk used on opensource projects

Example 1 with MeasureRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk in project carbondata by apache.

the class AbstractBlockletScanner method scanBlocklet.

@Override
public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException, FilterUnsupportedException {
    long startTime = System.currentTimeMillis();
    AbstractScannedResult scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
    QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
    totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, totalBlockletStatistic.getCount() + 1);
    QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
    validScannedBlockletStatistic.addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, validScannedBlockletStatistic.getCount() + 1);
    // adding statistics for valid number of pages
    QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
    validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
    // adding statistics for number of pages
    QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
    totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
    scannedResult.setBlockletId(blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder.getDataBlock().nodeNumber());
    DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDimensionRawDataChunk();
    DimensionColumnDataChunk[][] dimensionColumnDataChunks = new DimensionColumnDataChunk[dimensionRawColumnChunks.length][];
    for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
        if (dimensionRawColumnChunks[i] != null) {
            dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].convertToDimColDataChunks();
        }
    }
    scannedResult.setDimensionChunks(dimensionColumnDataChunks);
    MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getMeasureRawDataChunk();
    MeasureColumnDataChunk[][] measureColumnDataChunks = new MeasureColumnDataChunk[measureRawColumnChunks.length][];
    for (int i = 0; i < measureRawColumnChunks.length; i++) {
        if (measureRawColumnChunks[i] != null) {
            measureColumnDataChunks[i] = measureRawColumnChunks[i].convertToMeasureColDataChunks();
        }
    }
    scannedResult.setMeasureChunks(measureColumnDataChunks);
    int[] numberOfRows = new int[] { blocksChunkHolder.getDataBlock().nodeSize() };
    if (blockExecutionInfo.getAllSelectedDimensionBlocksIndexes().length > 0) {
        for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
            if (dimensionRawColumnChunks[i] != null) {
                numberOfRows = dimensionRawColumnChunks[i].getRowCount();
                break;
            }
        }
    } else if (blockExecutionInfo.getAllSelectedMeasureBlocksIndexes().length > 0) {
        for (int i = 0; i < measureRawColumnChunks.length; i++) {
            if (measureRawColumnChunks[i] != null) {
                numberOfRows = measureRawColumnChunks[i].getRowCount();
                break;
            }
        }
    }
    scannedResult.setNumberOfRows(numberOfRows);
    // loading delete data cache in blockexecutioninfo instance
    DeleteDeltaCacheLoaderIntf deleteCacheLoader = new BlockletDeleteDeltaCacheLoader(scannedResult.getBlockletId(), blocksChunkHolder.getDataBlock(), blockExecutionInfo.getAbsoluteTableIdentifier());
    deleteCacheLoader.loadDeleteDeltaFileDataToCache();
    scannedResult.setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
    scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
    // adding statistics for carbon scan time
    QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
    scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, scanTime.getCount() + (System.currentTimeMillis() - startTime));
    return scannedResult;
}

Example 2 with MeasureRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk in project carbondata by apache.

the class AbstractBlockletScanner method readBlocklet.

@Override
public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException {
    long startTime = System.currentTimeMillis();
    DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDataBlock().getDimensionChunks(blocksChunkHolder.getFileReader(), blockExecutionInfo.getAllSelectedDimensionBlocksIndexes());
    blocksChunkHolder.setDimensionRawDataChunk(dimensionRawColumnChunks);
    MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getDataBlock().getMeasureChunks(blocksChunkHolder.getFileReader(), blockExecutionInfo.getAllSelectedMeasureBlocksIndexes());
    blocksChunkHolder.setMeasureRawDataChunk(measureRawColumnChunks);
    // adding statistics for carbon read time
    QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
    readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME, readTime.getCount() + (System.currentTimeMillis() - startTime));
}

Example 3 with MeasureRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV3 method readRawMeasureChunk.

/**
   * Below method will be used to read the measure column data form carbon data file
   * 1. Get the length of the data to be read
   * 2. Allocate the direct buffer
   * 3. read the data from file
   * 4. Get the data chunk object from data read
   * 5. Create the raw chunk object and fill the details
   *
   * @param fileReader          reader for reading the column from carbon data file
   * @param blockIndex          blocklet index of the column in carbon data file
   * @return measure raw chunk
   */
@Override
public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int blockletColumnIndex) throws IOException {
    int dataLength = 0;
    // of the last dimension, we can subtract current dimension offset from lastDimesionOffset
    if (measureColumnChunkOffsets.size() - 1 == blockletColumnIndex) {
        dataLength = (int) (measureOffsets - measureColumnChunkOffsets.get(blockletColumnIndex));
    } else {
        dataLength = (int) (measureColumnChunkOffsets.get(blockletColumnIndex + 1) - measureColumnChunkOffsets.get(blockletColumnIndex));
    }
    ByteBuffer buffer = null;
    // read the data from carbon data file
    synchronized (fileReader) {
        buffer = fileReader.readByteBuffer(filePath, measureColumnChunkOffsets.get(blockletColumnIndex), dataLength);
    }
    // get the data chunk which will have all the details about the data pages
    DataChunk3 dataChunk = CarbonUtil.readDataChunk3(buffer, 0, measureColumnChunkLength.get(blockletColumnIndex));
    // creating a raw chunks instance and filling all the details
    MeasureRawColumnChunk rawColumnChunk = new MeasureRawColumnChunk(blockletColumnIndex, buffer, 0, dataLength, this);
    int numberOfPages = dataChunk.getPage_length().size();
    byte[][] maxValueOfEachPage = new byte[numberOfPages][];
    byte[][] minValueOfEachPage = new byte[numberOfPages][];
    int[] eachPageLength = new int[numberOfPages];
    for (int i = 0; i < minValueOfEachPage.length; i++) {
        maxValueOfEachPage[i] = dataChunk.getData_chunk_list().get(i).getMin_max().getMax_values().get(0).array();
        minValueOfEachPage[i] = dataChunk.getData_chunk_list().get(i).getMin_max().getMin_values().get(0).array();
        eachPageLength[i] = dataChunk.getData_chunk_list().get(i).getNumberOfRowsInpage();
    }
    rawColumnChunk.setDataChunkV3(dataChunk);
    rawColumnChunk.setFileReader(fileReader);
    rawColumnChunk.setPagesCount(dataChunk.getPage_length().size());
    rawColumnChunk.setMaxValues(maxValueOfEachPage);
    rawColumnChunk.setMinValues(minValueOfEachPage);
    rawColumnChunk.setRowCount(eachPageLength);
    rawColumnChunk.setLengths(ArrayUtils.toPrimitive(dataChunk.page_length.toArray(new Integer[dataChunk.page_length.size()])));
    rawColumnChunk.setOffsets(ArrayUtils.toPrimitive(dataChunk.page_offset.toArray(new Integer[dataChunk.page_offset.size()])));
    return rawColumnChunk;
}

Example 4 with MeasureRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV3 method readRawMeasureChunksInGroup.

/**
   * Below method will be used to read the multiple measure column data in group
   * and divide into measure raw chunk object
   * Steps for reading
   * 1. Get the length of the data to be read
   * 2. Allocate the direct buffer
   * 3. read the data from file
   * 4. Get the data chunk object from file for each column
   * 5. Create the raw chunk object and fill the details for each column
   * 6. increment the offset of the data
   *
   * @param fileReader
   *        reader which will be used to read the measure columns data from file
   * @param startColumnBlockletIndex
   *        blocklet index of the first measure column
   * @param endColumnBlockletIndex
   *        blocklet index of the last measure column
   * @return MeasureRawColumnChunk array
   */
protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader, int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
    // to calculate the length of the data to be read
    // column we can subtract the offset of start column offset with
    // end column+1 offset and get the total length.
    long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnBlockletIndex);
    ByteBuffer buffer = null;
    // read the data from carbon data file
    synchronized (fileReader) {
        buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset, (int) (measureColumnChunkOffsets.get(endColumnBlockletIndex + 1) - currentMeasureOffset));
    }
    // create raw chunk for each measure column
    MeasureRawColumnChunk[] measureDataChunk = new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1];
    int runningLength = 0;
    int index = 0;
    for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) {
        int currentLength = (int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i));
        MeasureRawColumnChunk measureRawColumnChunk = new MeasureRawColumnChunk(i, buffer, runningLength, currentLength, this);
        DataChunk3 dataChunk = CarbonUtil.readDataChunk3(buffer, runningLength, measureColumnChunkLength.get(i));
        int numberOfPages = dataChunk.getPage_length().size();
        byte[][] maxValueOfEachPage = new byte[numberOfPages][];
        byte[][] minValueOfEachPage = new byte[numberOfPages][];
        int[] eachPageLength = new int[numberOfPages];
        for (int j = 0; j < minValueOfEachPage.length; j++) {
            maxValueOfEachPage[j] = dataChunk.getData_chunk_list().get(j).getMin_max().getMax_values().get(0).array();
            minValueOfEachPage[j] = dataChunk.getData_chunk_list().get(j).getMin_max().getMin_values().get(0).array();
            eachPageLength[j] = dataChunk.getData_chunk_list().get(j).getNumberOfRowsInpage();
        }
        measureRawColumnChunk.setDataChunkV3(dataChunk);
        ;
        measureRawColumnChunk.setFileReader(fileReader);
        measureRawColumnChunk.setPagesCount(dataChunk.getPage_length().size());
        measureRawColumnChunk.setMaxValues(maxValueOfEachPage);
        measureRawColumnChunk.setMinValues(minValueOfEachPage);
        measureRawColumnChunk.setRowCount(eachPageLength);
        measureRawColumnChunk.setLengths(ArrayUtils.toPrimitive(dataChunk.page_length.toArray(new Integer[dataChunk.page_length.size()])));
        measureRawColumnChunk.setOffsets(ArrayUtils.toPrimitive(dataChunk.page_offset.toArray(new Integer[dataChunk.page_offset.size()])));
        measureDataChunk[index] = measureRawColumnChunk;
        runningLength += currentLength;
        index++;
    }
    return measureDataChunk;
}

Example 5 with MeasureRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk in project carbondata by apache.

the class ExcludeFilterExecuterImpl method applyFilter.

@Override
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeLine) throws IOException {
    if (isDimensionPresentInCurrentBlock) {
        int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(dimColEvaluatorInfo.getColumnIndex());
        if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
            rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
        }
        DimensionRawColumnChunk dimensionRawColumnChunk = rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
        DimensionColumnPage[] dimensionColumnPages = dimensionRawColumnChunk.decodeAllColumnPages();
        BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
        for (int i = 0; i < dimensionColumnPages.length; i++) {
            BitSet bitSet = getFilteredIndexes(dimensionColumnPages[i], dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine, rawBlockletColumnChunks.getBitSetGroup(), i);
            bitSetGroup.setBitSet(bitSet, i);
        }
        return bitSetGroup;
    } else if (isMeasurePresentInCurrentBlock) {
        int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping().get(msrColumnEvaluatorInfo.getColumnIndex());
        if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
            rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
        }
        MeasureRawColumnChunk measureRawColumnChunk = rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
        ColumnPage[] ColumnPages = measureRawColumnChunk.decodeAllColumnPages();
        BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount());
        DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo);
        for (int i = 0; i < ColumnPages.length; i++) {
            BitSet bitSet = getFilteredIndexesForMeasure(measureRawColumnChunk.decodeColumnPage(i), measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine, rawBlockletColumnChunks.getBitSetGroup(), i, msrType);
            bitSetGroup.setBitSet(bitSet, i);
        }
        return bitSetGroup;
    }
    return null;
}