Examples with MultiDimKeyVarLengthEquiSplitGenerator org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthEquiSplitGenerator used on opensource projects

Example 1 with MultiDimKeyVarLengthEquiSplitGenerator

use of org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthEquiSplitGenerator in project carbondata by apache.

the class CarbonFactDataHandlerColumnar method setWritingConfiguration.

/**
 * Below method will be to configure fact file writing configuration
 *
 * @throws CarbonDataWriterException
 */
private void setWritingConfiguration() throws CarbonDataWriterException {
    // get blocklet size
    this.pageSize = Integer.parseInt(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.BLOCKLET_SIZE, CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL));
    if (version == ColumnarFormatVersion.V3) {
        this.pageSize = CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
    }
    LOGGER.info("Number of rows per column blocklet " + pageSize);
    dataRows = new ArrayList<>(this.pageSize);
    int dimSet = Integer.parseInt(CarbonCommonConstants.DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE);
    // if at least one dimension is present then initialize column splitter otherwise null
    int noOfColStore = colGrpModel.getNoOfColumnStore();
    int[] keyBlockSize = new int[noOfColStore + getExpandedComplexColsCount()];
    if (model.getDimLens().length > 0) {
        // Using Variable length variable split generator
        // This will help in splitting mdkey to columns. variable split is required because all
        // columns which are part of
        // row store will be in single column store
        // e.g if {0,1,2,3,4,5} is dimension and {0,1,2) is row store dimension
        // than below splitter will return column as {0,1,2}{3}{4}{5}
        ColumnarSplitter columnarSplitter = model.getSegmentProperties().getFixedLengthKeySplitter();
        System.arraycopy(columnarSplitter.getBlockKeySize(), 0, keyBlockSize, 0, noOfColStore);
    }
    // agg type
    List<Integer> otherMeasureIndexList = new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<Integer> customMeasureIndexList = new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    DataType[] type = model.getMeasureDataType();
    for (int j = 0; j < type.length; j++) {
        if (type[j] != DataTypes.BYTE && !DataTypes.isDecimal(type[j])) {
            otherMeasureIndexList.add(j);
        } else {
            customMeasureIndexList.add(j);
        }
    }
    int[] otherMeasureIndex = new int[otherMeasureIndexList.size()];
    int[] customMeasureIndex = new int[customMeasureIndexList.size()];
    for (int i = 0; i < otherMeasureIndex.length; i++) {
        otherMeasureIndex[i] = otherMeasureIndexList.get(i);
    }
    for (int i = 0; i < customMeasureIndex.length; i++) {
        customMeasureIndex[i] = customMeasureIndexList.get(i);
    }
    setComplexMapSurrogateIndex(model.getDimensionCount());
    int[] blockKeySize = getBlockKeySizeWithComplexTypes(new MultiDimKeyVarLengthEquiSplitGenerator(CarbonUtil.getIncrementedCardinalityFullyFilled(model.getDimLens().clone()), (byte) dimSet).getBlockKeySize());
    System.arraycopy(blockKeySize, noOfColStore, keyBlockSize, noOfColStore, blockKeySize.length - noOfColStore);
    this.dataWriter = getFactDataWriter();
    // initialize the channel;
    this.dataWriter.initializeWriter();
}