Examples with MeasureColumnDataChunk org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk used on opensource projects

Example 1 with MeasureColumnDataChunk

use of org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk in project carbondata by apache.

the class AbstractBlockletScanner method scanBlocklet.

@Override
public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException, FilterUnsupportedException {
    long startTime = System.currentTimeMillis();
    AbstractScannedResult scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
    QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
    totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, totalBlockletStatistic.getCount() + 1);
    QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
    validScannedBlockletStatistic.addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, validScannedBlockletStatistic.getCount() + 1);
    // adding statistics for valid number of pages
    QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
    validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
    // adding statistics for number of pages
    QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
    totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
    scannedResult.setBlockletId(blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder.getDataBlock().nodeNumber());
    DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDimensionRawDataChunk();
    DimensionColumnDataChunk[][] dimensionColumnDataChunks = new DimensionColumnDataChunk[dimensionRawColumnChunks.length][];
    for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
        if (dimensionRawColumnChunks[i] != null) {
            dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].convertToDimColDataChunks();
        }
    }
    scannedResult.setDimensionChunks(dimensionColumnDataChunks);
    MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getMeasureRawDataChunk();
    MeasureColumnDataChunk[][] measureColumnDataChunks = new MeasureColumnDataChunk[measureRawColumnChunks.length][];
    for (int i = 0; i < measureRawColumnChunks.length; i++) {
        if (measureRawColumnChunks[i] != null) {
            measureColumnDataChunks[i] = measureRawColumnChunks[i].convertToMeasureColDataChunks();
        }
    }
    scannedResult.setMeasureChunks(measureColumnDataChunks);
    int[] numberOfRows = new int[] { blocksChunkHolder.getDataBlock().nodeSize() };
    if (blockExecutionInfo.getAllSelectedDimensionBlocksIndexes().length > 0) {
        for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
            if (dimensionRawColumnChunks[i] != null) {
                numberOfRows = dimensionRawColumnChunks[i].getRowCount();
                break;
            }
        }
    } else if (blockExecutionInfo.getAllSelectedMeasureBlocksIndexes().length > 0) {
        for (int i = 0; i < measureRawColumnChunks.length; i++) {
            if (measureRawColumnChunks[i] != null) {
                numberOfRows = measureRawColumnChunks[i].getRowCount();
                break;
            }
        }
    }
    scannedResult.setNumberOfRows(numberOfRows);
    // loading delete data cache in blockexecutioninfo instance
    DeleteDeltaCacheLoaderIntf deleteCacheLoader = new BlockletDeleteDeltaCacheLoader(scannedResult.getBlockletId(), blocksChunkHolder.getDataBlock(), blockExecutionInfo.getAbsoluteTableIdentifier());
    deleteCacheLoader.loadDeleteDeltaFileDataToCache();
    scannedResult.setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
    scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
    // adding statistics for carbon scan time
    QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
    scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, scanTime.getCount() + (System.currentTimeMillis() - startTime));
    return scannedResult;
}

Example 2 with MeasureColumnDataChunk

use of org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV2 method convertToMeasureChunk.

public MeasureColumnDataChunk convertToMeasureChunk(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException {
    MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
    DataChunk2 measureColumnChunk = null;
    int copyPoint = measureRawColumnChunk.getOffSet();
    int blockIndex = measureRawColumnChunk.getBlockletId();
    ByteBuffer rawData = measureRawColumnChunk.getRawData();
    if (measureColumnChunkOffsets.size() - 1 == blockIndex) {
        measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint, measureColumnChunkLength.get(blockIndex));
        synchronized (measureRawColumnChunk.getFileReader()) {
            rawData = measureRawColumnChunk.getFileReader().readByteBuffer(filePath, measureColumnChunkOffsets.get(blockIndex) + measureColumnChunkLength.get(blockIndex), measureColumnChunk.data_page_length);
        }
    } else {
        measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint, measureColumnChunkLength.get(blockIndex));
        copyPoint += measureColumnChunkLength.get(blockIndex);
    }
    List<ValueEncoderMeta> valueEncodeMeta = new ArrayList<>();
    for (int i = 0; i < measureColumnChunk.getEncoder_meta().size(); i++) {
        valueEncodeMeta.add(CarbonUtil.deserializeEncoderMeta(measureColumnChunk.getEncoder_meta().get(i).array()));
    }
    WriterCompressModel compressionModel = CarbonUtil.getValueCompressionModel(valueEncodeMeta);
    ValueCompressionHolder values = compressionModel.getValueCompressionHolder()[0];
    // uncompress
    values.uncompress(compressionModel.getConvertedDataType()[0], rawData.array(), copyPoint, measureColumnChunk.data_page_length, compressionModel.getMantissa()[0], compressionModel.getMaxValue()[0], numberOfRows);
    CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
    // set the data chunk
    datChunk.setMeasureDataHolder(measureDataHolder);
    // set the enun value indexes
    datChunk.setNullValueIndexHolder(getPresenceMeta(measureColumnChunk.presence));
    return datChunk;
}

Example 3 with MeasureColumnDataChunk

use of org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV3 method convertToMeasureChunk.

/**
   * Below method will be used to convert the compressed measure chunk raw data to actual data
   *
   * @param measureRawColumnChunk measure raw chunk
   * @param pageNumber            number
   * @return DimensionColumnDataChunk
   */
@Override
public MeasureColumnDataChunk convertToMeasureChunk(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException {
    MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
    // data chunk of blocklet column
    DataChunk3 dataChunk3 = measureRawColumnChunk.getDataChunkV3();
    // data chunk of page
    DataChunk2 measureColumnChunk = dataChunk3.getData_chunk_list().get(pageNumber);
    // calculating the start point of data
    // as buffer can contain multiple column data, start point will be datachunkoffset +
    // data chunk length + page offset
    int copyPoint = measureRawColumnChunk.getOffSet() + measureColumnChunkLength.get(measureRawColumnChunk.getBlockletId()) + dataChunk3.getPage_offset().get(pageNumber);
    List<ValueEncoderMeta> valueEncodeMeta = new ArrayList<>();
    for (int i = 0; i < measureColumnChunk.getEncoder_meta().size(); i++) {
        valueEncodeMeta.add(CarbonUtil.deserializeEncoderMetaNew(measureColumnChunk.getEncoder_meta().get(i).array()));
    }
    WriterCompressModel compressionModel = CarbonUtil.getValueCompressionModel(valueEncodeMeta);
    ValueCompressionHolder values = compressionModel.getValueCompressionHolder()[0];
    // uncompress
    ByteBuffer rawData = measureRawColumnChunk.getRawData();
    values.uncompress(compressionModel.getConvertedDataType()[0], rawData.array(), copyPoint, measureColumnChunk.data_page_length, compressionModel.getMantissa()[0], compressionModel.getMaxValue()[0], measureRawColumnChunk.getRowCount()[pageNumber]);
    CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
    // set the data chunk
    datChunk.setMeasureDataHolder(measureDataHolder);
    // set the null value indexes
    datChunk.setNullValueIndexHolder(getPresenceMeta(measureColumnChunk.presence));
    return datChunk;
}

Example 4 with MeasureColumnDataChunk

use of org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk in project carbondata by apache.

the class RowLevelFilterExecuterImpl method createRow.

/**
   * Method will read the members of particular dimension block and create
   * a row instance for further processing of the filters
   *
   * @param blockChunkHolder
   * @param row
   * @param index
   * @throws IOException
   */
private void createRow(BlocksChunkHolder blockChunkHolder, RowIntf row, int pageIndex, int index) throws IOException {
    Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
    String memberString;
    for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
        DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
        // if filter dimension is not present in the current add its default value
        if (!isDimensionPresentInCurrentBlock[i]) {
            // fill default value here
            record[dimColumnEvaluatorInfo.getRowIndex()] = getDimensionDefaultValue(dimColumnEvaluatorInfo);
            continue;
        }
        if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
            if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
                record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDimension().getDefaultValue();
            }
            DimensionColumnDataChunk columnDataChunk = blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]].convertToDimColDataChunk(pageIndex);
            if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && columnDataChunk instanceof VariableLengthDimensionDataChunk) {
                VariableLengthDimensionDataChunk dimensionColumnDataChunk = (VariableLengthDimensionDataChunk) columnDataChunk;
                byte[] memberBytes = dimensionColumnDataChunk.getChunkData(index);
                if (null != memberBytes) {
                    if (Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, memberBytes)) {
                        memberBytes = null;
                    }
                    record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes, dimColumnEvaluatorInfo.getDimension().getDataType());
                } else {
                    continue;
                }
            } else {
                int dictionaryValue = readSurrogatesFromColumnBlock(blockChunkHolder, index, pageIndex, dimColumnEvaluatorInfo, dimensionBlocksIndex[i]);
                if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    memberString = getFilterActualValueFromDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
                    record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil.getDataBasedOnDataType(memberString, dimColumnEvaluatorInfo.getDimension().getDataType());
                } else if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    Object member = getFilterActualValueFromDirectDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
                    record[dimColumnEvaluatorInfo.getRowIndex()] = member;
                }
            }
        } else {
            try {
                GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
                ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
                DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
                complexType.parseBlocksAndReturnComplexColumnByteArray(blockChunkHolder.getDimensionRawDataChunk(), index, pageIndex, dataOutputStream);
                record[dimColumnEvaluatorInfo.getRowIndex()] = complexType.getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
                byteStream.close();
            } catch (IOException e) {
                LOGGER.info(e.getMessage());
            }
        }
    }
    DataType msrType;
    for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
        MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i);
        switch(msrColumnEvalutorInfo.getType()) {
            case SHORT:
                msrType = DataType.SHORT;
                break;
            case INT:
                msrType = DataType.INT;
                break;
            case LONG:
                msrType = DataType.LONG;
                break;
            case DECIMAL:
                msrType = DataType.DECIMAL;
                break;
            default:
                msrType = DataType.DOUBLE;
        }
        // in the current block measure list
        if (!isMeasurePresentInCurrentBlock[i]) {
            byte[] defaultValue = msrColumnEvalutorInfo.getCarbonColumn().getDefaultValue();
            record[msrColumnEvalutorInfo.getRowIndex()] = RestructureUtil.getMeasureDefaultValue(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema(), defaultValue);
            continue;
        }
        Object msrValue;
        MeasureColumnDataChunk measureColumnDataChunk = blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].convertToMeasureColDataChunk(pageIndex);
        switch(msrType) {
            case SHORT:
                msrValue = (short) measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
                break;
            case INT:
                msrValue = (int) measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
                break;
            case LONG:
                msrValue = measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
                break;
            case DECIMAL:
                BigDecimal bigDecimalValue = measureColumnDataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
                if (null != bigDecimalValue && msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale() > bigDecimalValue.scale()) {
                    bigDecimalValue = bigDecimalValue.setScale(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale(), RoundingMode.HALF_UP);
                }
                msrValue = bigDecimalValue;
                break;
            default:
                msrValue = measureColumnDataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
        }
        record[msrColumnEvalutorInfo.getRowIndex()] = measureColumnDataChunk.getNullValueIndexHolder().getBitSet().get(index) ? null : msrValue;
    }
    row.setValues(record);
}

Example 5 with MeasureColumnDataChunk

use of org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV1 method convertToMeasureChunk.

@Override
public MeasureColumnDataChunk convertToMeasureChunk(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException {
    int blockIndex = measureRawColumnChunk.getBlockletId();
    DataChunk dataChunk = measureColumnChunks.get(blockIndex);
    ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0);
    ReaderCompressModel compressModel = ValueCompressionUtil.getReaderCompressModel(meta);
    ValueCompressionHolder values = compressModel.getValueCompressionHolder();
    ByteBuffer rawData = measureRawColumnChunk.getRawData();
    // unCompress data
    values.uncompress(compressModel.getConvertedDataType(), rawData.array(), measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength(), compressModel.getMantissa(), compressModel.getMaxValue(), numberOfRows);
    CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
    // create and set the data chunk
    MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
    datChunk.setMeasureDataHolder(measureDataHolder);
    // set the enun value indexes
    datChunk.setNullValueIndexHolder(dataChunk.getNullValueIndexForColumn());
    return datChunk;
}