Examples with MultiDimKeyVarLengthVariableSplitGenerator org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator used on opensource projects

Example 1 with MultiDimKeyVarLengthVariableSplitGenerator

use of org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator in project carbondata by apache.

the class SegmentProperties method fillKeyGeneratorDetails.

/**
   * Below method will fill the key generator detail of both the type of key
   * generator. This will be required for during both query execution and data
   * loading.
   */
private void fillKeyGeneratorDetails() {
    // create a dimension partitioner list
    // this list will contain information about how dimension value are
    // stored
    // it is stored in group or individually
    List<Integer> dimensionPartitionList = new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<Boolean> isDictionaryColumn = new ArrayList<Boolean>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    int prvcolumnGroupId = -1;
    int counter = 0;
    while (counter < dimensions.size()) {
        CarbonDimension carbonDimension = dimensions.get(counter);
        // if dimension is not a part of mdkey then no need to add
        if (!carbonDimension.getEncoder().contains(Encoding.DICTIONARY)) {
            isDictionaryColumn.add(false);
            counter++;
            continue;
        }
        // so add one
        if (carbonDimension.isColumnar()) {
            dimensionPartitionList.add(1);
            isDictionaryColumn.add(true);
        }
        // group
        if (!carbonDimension.isColumnar() && carbonDimension.columnGroupId() == prvcolumnGroupId) {
            // incrementing the previous value of the list as it is in same column group
            dimensionPartitionList.set(dimensionPartitionList.size() - 1, dimensionPartitionList.get(dimensionPartitionList.size() - 1) + 1);
        } else if (!carbonDimension.isColumnar()) {
            dimensionPartitionList.add(1);
            isDictionaryColumn.add(true);
        }
        prvcolumnGroupId = carbonDimension.columnGroupId();
        counter++;
    }
    // get the partitioner
    dimensionPartitions = ArrayUtils.toPrimitive(dimensionPartitionList.toArray(new Integer[dimensionPartitionList.size()]));
    // get the bit length of each column
    int[] bitLength = CarbonUtil.getDimensionBitLength(dimColumnsCardinality, dimensionPartitions);
    // create a key generator
    this.dimensionKeyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
    if (this.getNumberOfDictSortColumns() == bitLength.length) {
        this.sortColumnsGenerator = this.dimensionKeyGenerator;
    } else {
        int numberOfDictSortColumns = this.getNumberOfDictSortColumns();
        int[] sortColumnBitLength = new int[numberOfDictSortColumns];
        System.arraycopy(bitLength, 0, sortColumnBitLength, 0, numberOfDictSortColumns);
        this.sortColumnsGenerator = new MultiDimKeyVarLengthGenerator(sortColumnBitLength);
    }
    this.fixedLengthKeySplitter = new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, dimensionPartitions);
    // get the size of each value in file block
    int[] dictionayDimColumnValueSize = fixedLengthKeySplitter.getBlockKeySize();
    int index = -1;
    this.eachDimColumnValueSize = new int[isDictionaryColumn.size()];
    for (int i = 0; i < eachDimColumnValueSize.length; i++) {
        if (!isDictionaryColumn.get(i)) {
            eachDimColumnValueSize[i] = -1;
            continue;
        }
        eachDimColumnValueSize[i] = dictionayDimColumnValueSize[++index];
    }
    if (complexDimensions.size() > 0) {
        int[] complexDimesionParition = new int[complexDimColumnCardinality.length];
        // as complex dimension will be stored in column format add one
        Arrays.fill(complexDimesionParition, 1);
        bitLength = CarbonUtil.getDimensionBitLength(complexDimColumnCardinality, complexDimesionParition);
        for (int i = 0; i < bitLength.length; i++) {
            if (complexDimColumnCardinality[i] == 0) {
                bitLength[i] = 64;
            }
        }
        ColumnarSplitter keySplitter = new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, complexDimesionParition);
        eachComplexDimColumnValueSize = keySplitter.getBlockKeySize();
    } else {
        eachComplexDimColumnValueSize = new int[0];
    }
}