Examples with MultiDimKeyVarLengthGenerator org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator used on opensource projects

Example 11 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class BTreeBlockFinderTest method getFileFooterListWithOnlyNoDictionaryKey.

private List<DataFileFooter> getFileFooterListWithOnlyNoDictionaryKey() {
    List<DataFileFooter> list = new ArrayList<DataFileFooter>();
    try {
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
        KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
        int i = 1;
        while (i < 1001) {
            byte[] startKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i, i });
            byte[] endKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i + 10, i + 10 });
            ByteBuffer buffer = ByteBuffer.allocate(2 + 4);
            buffer.rewind();
            buffer.putShort((short) 1);
            buffer.putInt(i);
            buffer.array();
            byte[] noDictionaryStartKey = buffer.array();
            ByteBuffer buffer1 = ByteBuffer.allocate(2 + 4);
            buffer1.rewind();
            buffer1.putShort((short) 2);
            buffer1.putInt(i + 10);
            buffer1.array();
            byte[] noDictionaryEndKey = buffer1.array();
            DataFileFooter footer = getFileMatadataWithOnlyNoDictionaryKey(startKey, endKey, noDictionaryStartKey, noDictionaryEndKey);
            list.add(footer);
            i = i + 10;
        }
    } catch (Exception e) {
        LOGGER.error(e);
    }
    return list;
}

Example 12 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class RestructureBasedRawResultCollector method initCurrentBlockKeyGenerator.

/**
 * This method will initialize the block key generator for the current block based on the
 * dictionary columns present in the current block
 */
private void initCurrentBlockKeyGenerator() {
    SegmentProperties segmentProperties = executionInfo.getDataBlock().getSegmentProperties();
    int[] dictionaryColumnBlockIndex = executionInfo.getDictionaryColumnChunkIndex();
    int[] updatedColumnCardinality = new int[dictionaryColumnBlockIndex.length];
    int[] updatedDimensionPartitioner = new int[dictionaryColumnBlockIndex.length];
    for (int i = 0; i < dictionaryColumnBlockIndex.length; i++) {
        // get the dictionary key ordinal as column cardinality in segment properties
        // will only be for dictionary encoded columns
        CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dictionaryColumnBlockIndex[i]);
        updatedColumnCardinality[i] = segmentProperties.getDimColumnsCardinality()[currentBlockDimension.getKeyOrdinal()];
        updatedDimensionPartitioner[i] = segmentProperties.getDimensionPartitions()[currentBlockDimension.getKeyOrdinal()];
    }
    if (dictionaryColumnBlockIndex.length > 0) {
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(updatedColumnCardinality, updatedDimensionPartitioner);
        updatedCurrentBlockKeyGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    }
}

Example 13 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class RestructureBasedRawResultCollector method initRestructuredKeyGenerator.

/**
 * This method will create a new key generator for generating mdKey according to latest schema
 */
private void initRestructuredKeyGenerator() {
    SegmentProperties segmentProperties = executionInfo.getDataBlock().getSegmentProperties();
    ProjectionDimension[] queryDimensions = executionInfo.getActualQueryDimensions();
    List<Integer> updatedColumnCardinality = new ArrayList<>(queryDimensions.length);
    List<Integer> updatedDimensionPartitioner = new ArrayList<>(queryDimensions.length);
    int[] dictionaryColumnBlockIndex = executionInfo.getDictionaryColumnChunkIndex();
    int dimCounterInCurrentBlock = 0;
    for (int i = 0; i < queryDimensions.length; i++) {
        if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
            if (executionInfo.getDimensionInfo().getDimensionExists()[i]) {
                // get the dictionary key ordinal as column cardinality in segment properties
                // will only be for dictionary encoded columns
                CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dictionaryColumnBlockIndex[dimCounterInCurrentBlock]);
                updatedColumnCardinality.add(segmentProperties.getDimColumnsCardinality()[currentBlockDimension.getKeyOrdinal()]);
                updatedDimensionPartitioner.add(segmentProperties.getDimensionPartitions()[currentBlockDimension.getKeyOrdinal()]);
                dimCounterInCurrentBlock++;
            } else {
                // partitioner index will be 1 every column will be in columnar format
                updatedDimensionPartitioner.add(1);
                // for direct dictionary 4 bytes need to be allocated else 1
                if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    updatedColumnCardinality.add(Integer.MAX_VALUE);
                } else {
                    // cardinality will be 2 will user has provided a default value
                    byte[] defaultValue = queryDimensions[i].getDimension().getDefaultValue();
                    if (null != defaultValue) {
                        updatedColumnCardinality.add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY + 1);
                    } else {
                        updatedColumnCardinality.add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY);
                    }
                }
            }
        }
    }
    if (!updatedColumnCardinality.isEmpty()) {
        int[] latestColumnCardinality = ArrayUtils.toPrimitive(updatedColumnCardinality.toArray(new Integer[updatedColumnCardinality.size()]));
        int[] latestColumnPartitioner = ArrayUtils.toPrimitive(updatedDimensionPartitioner.toArray(new Integer[updatedDimensionPartitioner.size()]));
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(latestColumnCardinality, latestColumnPartitioner);
        restructuredKeyGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    }
}