Examples with ColumnPageWrapper org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper used on opensource projects

Example 1 with ColumnPageWrapper

use of org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper in project carbondata by apache.

the class DimensionChunkReaderV3 method decodeDimension.

protected DimensionColumnPage decodeDimension(DimensionRawColumnChunk rawColumnPage, ByteBuffer pageData, DataChunk2 pageMetadata, int offset, ColumnVectorInfo vectorInfo, ReusableDataBuffer reusableDataBuffer) throws IOException {
    List<Encoding> encodings = pageMetadata.getEncoders();
    org.apache.carbondata.core.metadata.encoder.Encoding.validateEncodingTypes(encodings);
    if (CarbonUtil.isEncodedWithMeta(encodings)) {
        int[] invertedIndexes = new int[0];
        int[] invertedIndexesReverse = new int[0];
        // in case of no dictionary measure data types, if it is included in sort columns
        // then inverted index to be uncompressed
        boolean isExplicitSorted = CarbonUtil.hasEncoding(pageMetadata.encoders, Encoding.INVERTED_INDEX);
        int dataOffset = offset;
        if (isExplicitSorted) {
            offset += pageMetadata.data_page_length;
            invertedIndexes = CarbonUtil.getUnCompressColumnIndex(pageMetadata.rowid_page_length, pageData, offset);
            if (vectorInfo == null) {
                // get the reverse index
                invertedIndexesReverse = CarbonUtil.getInvertedReverseIndex(invertedIndexes);
            } else {
                vectorInfo.invertedIndex = invertedIndexes;
            }
        }
        BitSet nullBitSet = QueryUtil.getNullBitSet(pageMetadata.presence, this.compressor);
        // Store local dictionary from rawColumnPage so it can be used while filling the vector
        if (vectorInfo != null && !vectorInfo.vectorStack.isEmpty() && rawColumnPage.getLocalDictionary() != null) {
            ((CarbonColumnVectorImpl) (vectorInfo.vectorStack.peek().getColumnVector())).setLocalDictionary(rawColumnPage.getLocalDictionary());
        }
        ColumnPage decodedPage = decodeDimensionByMeta(pageMetadata, pageData, dataOffset, null != rawColumnPage.getLocalDictionary(), vectorInfo, nullBitSet, reusableDataBuffer);
        if (decodedPage != null) {
            decodedPage.setNullBits(nullBitSet);
        }
        return new ColumnPageWrapper(decodedPage, rawColumnPage.getLocalDictionary(), invertedIndexes, invertedIndexesReverse, isEncodedWithAdaptiveMeta(pageMetadata), isExplicitSorted);
    } else {
        // following code is for backward compatibility
        return decodeDimensionLegacy(rawColumnPage, pageData, pageMetadata, offset, vectorInfo, reusableDataBuffer);
    }
}

Example 2 with ColumnPageWrapper

use of org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper in project carbondata by apache.

the class RowLevelFilterExecutorImpl method createRow.

/**
 * Method will read the members of particular dimension block and create
 * a row instance for further processing of the filters
 *
 * @param blockChunkHolder
 * @param row
 * @param index
 */
private void createRow(RawBlockletColumnChunks blockChunkHolder, RowIntf row, int pageIndex, int index) {
    Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
    String memberString;
    for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
        DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
        // if filter dimension is not present in the current add its default value
        if (!isDimensionPresentInCurrentBlock[i]) {
            // fill default value here
            record[dimColumnEvaluatorInfo.getRowIndex()] = getDimensionDefaultValue(dimColumnEvaluatorInfo);
            continue;
        }
        if (!dimColumnEvaluatorInfo.getDimension().getDataType().isComplexType()) {
            if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSlice()) {
                record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDimension().getDefaultValue();
            }
            DimensionColumnPage columnDataChunk = blockChunkHolder.getDimensionRawColumnChunks()[dimensionChunkIndex[i]].decodeColumnPage(pageIndex);
            if (dimColumnEvaluatorInfo.getDimension().getDataType() == DataTypes.DATE) {
                record[dimColumnEvaluatorInfo.getRowIndex()] = dateDictionaryGenerator.getValueFromSurrogate(ByteUtil.toInt(columnDataChunk.getChunkData(index), 0));
            } else if (columnDataChunk instanceof VariableLengthDimensionColumnPage || columnDataChunk instanceof ColumnPageWrapper) {
                byte[] memberBytes = columnDataChunk.getChunkData(index);
                if (null != memberBytes) {
                    if (Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, memberBytes)) {
                        memberBytes = null;
                    } else if (memberBytes.length == 0) {
                        memberBytes = null;
                    }
                    record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes, dimColumnEvaluatorInfo.getDimension().getDataType());
                }
            }
        } else {
            try {
                GenericQueryType complexType = complexDimensionInfoMap.get(dimensionChunkIndex[i]);
                ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
                DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
                complexType.parseBlocksAndReturnComplexColumnByteArray(blockChunkHolder.getDimensionRawColumnChunks(), null, index, pageIndex, dataOutputStream);
                record[dimColumnEvaluatorInfo.getRowIndex()] = complexType.getDataBasedOnDataType(ByteBuffer.wrap(byteStream.toByteArray()));
                byteStream.close();
            } catch (IOException e) {
                LOGGER.info(e.getMessage());
            }
        }
    }
    DataType msrType;
    for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
        MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i);
        DataType dataType = msrColumnEvalutorInfo.getType();
        if (dataType == DataTypes.BOOLEAN) {
            msrType = DataTypes.BOOLEAN;
        } else if (dataType == DataTypes.SHORT) {
            msrType = DataTypes.SHORT;
        } else if (dataType == DataTypes.INT) {
            msrType = DataTypes.INT;
        } else if (dataType == DataTypes.LONG) {
            msrType = DataTypes.LONG;
        } else if (DataTypes.isDecimal(dataType)) {
            msrType = DataTypes.createDefaultDecimalType();
        } else {
            msrType = DataTypes.DOUBLE;
        }
        // in the current block measure list
        if (!isMeasurePresentInCurrentBlock[i]) {
            byte[] defaultValue = msrColumnEvalutorInfo.getCarbonColumn().getDefaultValue();
            record[msrColumnEvalutorInfo.getRowIndex()] = RestructureUtil.getMeasureDefaultValue(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema(), defaultValue);
            continue;
        }
        Object msrValue;
        ColumnPage columnPage = blockChunkHolder.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0).getColumnIndex()].decodeColumnPage(pageIndex);
        if (msrType == DataTypes.BOOLEAN) {
            msrValue = columnPage.getBoolean(index);
        } else if (msrType == DataTypes.SHORT) {
            msrValue = (short) columnPage.getLong(index);
        } else if (msrType == DataTypes.INT) {
            msrValue = (int) columnPage.getLong(index);
        } else if (msrType == DataTypes.LONG) {
            msrValue = columnPage.getLong(index);
        } else if (DataTypes.isDecimal(msrType)) {
            BigDecimal bigDecimalValue = columnPage.getDecimal(index);
            if (null != bigDecimalValue && msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale() > bigDecimalValue.scale()) {
                bigDecimalValue = bigDecimalValue.setScale(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale(), RoundingMode.HALF_UP);
            }
            msrValue = bigDecimalValue;
        } else {
            msrValue = columnPage.getDouble(index);
        }
        record[msrColumnEvalutorInfo.getRowIndex()] = columnPage.getNullBits().get(index) ? null : msrValue;
    }
    row.setValues(record);
}