Examples with MultiDimKeyVarLengthGenerator org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator used on opensource projects

Example 6 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class BTreeBlockFinderTest method getFileFooterListWithOnlyDictionaryKey.

private List<DataFileFooter> getFileFooterListWithOnlyDictionaryKey() {
    List<DataFileFooter> list = new ArrayList<DataFileFooter>();
    try {
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
        KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
        int i = 1;
        while (i < 1001) {
            byte[] startKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i, i });
            byte[] endKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i + 10, i + 10 });
            ByteBuffer buffer = ByteBuffer.allocate(1 + 4);
            buffer.rewind();
            buffer.put((byte) 1);
            buffer.putInt(i);
            buffer.array();
            byte[] noDictionaryStartKey = buffer.array();
            ByteBuffer buffer1 = ByteBuffer.allocate(1 + 4);
            buffer1.rewind();
            buffer1.put((byte) 1);
            buffer1.putInt(i + 10);
            buffer1.array();
            byte[] noDictionaryEndKey = buffer1.array();
            DataFileFooter footer = getFileFooterWithOnlyDictionaryKey(startKey, endKey, noDictionaryStartKey, noDictionaryEndKey);
            list.add(footer);
            i = i + 10;
        }
    } catch (Exception e) {
        LOGGER.error(e);
    }
    return list;
}

Example 7 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class FilterUtilTest method testGetMaskKey.

@Test
public void testGetMaskKey() {
    int surrogate = 1;
    int[] keys = new int[] { 1, 2 };
    MultiDimKeyVarLengthGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(keys);
    int ordinal = 1;
    int keyOrdinal = 1;
    int columnGroupOrdinal = 1;
    int complexTypeOrdinal = 1;
    ColumnSchema columnSchema = new ColumnSchema();
    columnSchema.setColumnar(true);
    columnSchema.setColumnName("IMEI");
    columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema.setDataType(DataTypes.STRING);
    columnSchema.setDimensionColumn(true);
    CarbonDimension carbonDimension = new CarbonDimension(columnSchema, ordinal, keyOrdinal, columnGroupOrdinal, complexTypeOrdinal);
    byte[] expectedResult = new byte[] { 1 };
    byte[] actualResult = FilterUtil.getMaskKey(surrogate, carbonDimension, multiDimKeyVarLengthGenerator);
    assertArrayEquals(expectedResult, actualResult);
}

Example 8 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class ColumnGroupDimensionDataChunkTest method setup.

@BeforeClass
public static void setup() {
    int[] bitLength = CarbonUtil.getDimensionBitLength(new int[] { 10, 10, 10 }, new int[] { 3 });
    // create a key generator
    keyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
    byte[] data = new byte[keyGenerator.getKeySizeInBytes() * 3];
    int position = 0;
    for (int i = 1; i <= 3; i++) {
        try {
            System.arraycopy(keyGenerator.generateKey(new int[] { i, i, i }), 0, data, position, keyGenerator.getKeySizeInBytes());
        } catch (KeyGenException e) {
            assertTrue(false);
        }
        position += keyGenerator.getKeySizeInBytes();
    }
    columnGroupDimensionDataChunk = new ColumnGroupDimensionColumnPage(data, keyGenerator.getKeySizeInBytes(), 3);
}

Example 9 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class FilterUtilTest method testCreateIndexKeyFromResolvedFilterVal.

@Test
public void testCreateIndexKeyFromResolvedFilterVal() throws Exception {
    long[] startOrEndKey = new long[] { 0, 10 };
    byte[] startOrEndKeyForNoDictDimension = { 1, 2 };
    int[] keys = new int[] { 1, 2 };
    MultiDimKeyVarLengthGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(keys);
    assertTrue(FilterUtil.createIndexKeyFromResolvedFilterVal(startOrEndKey, multiDimKeyVarLengthGenerator, startOrEndKeyForNoDictDimension) != null);
}

Example 10 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class SegmentProperties method fillKeyGeneratorDetails.

/**
 * Below method will fill the key generator detail of both the type of key
 * generator. This will be required for during both query execution and data
 * loading.
 */
private void fillKeyGeneratorDetails() {
    // create a dimension partitioner list
    // this list will contain information about how dimension value are
    // stored
    // it is stored in group or individually
    List<Integer> dimensionPartitionList = new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<Boolean> isDictionaryColumn = new ArrayList<Boolean>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    int prvcolumnGroupId = -1;
    int counter = 0;
    while (counter < dimensions.size()) {
        CarbonDimension carbonDimension = dimensions.get(counter);
        // if dimension is not a part of mdkey then no need to add
        if (!carbonDimension.getEncoder().contains(Encoding.DICTIONARY)) {
            isDictionaryColumn.add(false);
            counter++;
            continue;
        }
        // so add one
        if (carbonDimension.isColumnar()) {
            dimensionPartitionList.add(1);
            isDictionaryColumn.add(true);
        }
        // group
        if (!carbonDimension.isColumnar() && carbonDimension.columnGroupId() == prvcolumnGroupId) {
            // incrementing the previous value of the list as it is in same column group
            dimensionPartitionList.set(dimensionPartitionList.size() - 1, dimensionPartitionList.get(dimensionPartitionList.size() - 1) + 1);
        } else if (!carbonDimension.isColumnar()) {
            dimensionPartitionList.add(1);
            isDictionaryColumn.add(true);
        }
        prvcolumnGroupId = carbonDimension.columnGroupId();
        counter++;
    }
    // get the partitioner
    dimensionPartitions = ArrayUtils.toPrimitive(dimensionPartitionList.toArray(new Integer[dimensionPartitionList.size()]));
    // get the bit length of each column
    int[] bitLength = CarbonUtil.getDimensionBitLength(dimColumnsCardinality, dimensionPartitions);
    // create a key generator
    this.dimensionKeyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
    if (this.getNumberOfDictSortColumns() == bitLength.length) {
        this.sortColumnsGenerator = this.dimensionKeyGenerator;
    } else {
        int numberOfDictSortColumns = this.getNumberOfDictSortColumns();
        int[] sortColumnBitLength = new int[numberOfDictSortColumns];
        System.arraycopy(bitLength, 0, sortColumnBitLength, 0, numberOfDictSortColumns);
        this.sortColumnsGenerator = new MultiDimKeyVarLengthGenerator(sortColumnBitLength);
    }
    this.fixedLengthKeySplitter = new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, dimensionPartitions);
    // get the size of each value in file block
    int[] dictionaryDimColumnValueSize = fixedLengthKeySplitter.getBlockKeySize();
    int index = -1;
    this.eachDimColumnValueSize = new int[isDictionaryColumn.size()];
    for (int i = 0; i < eachDimColumnValueSize.length; i++) {
        if (!isDictionaryColumn.get(i)) {
            eachDimColumnValueSize[i] = -1;
            continue;
        }
        eachDimColumnValueSize[i] = dictionaryDimColumnValueSize[++index];
    }
    if (complexDimensions.size() > 0) {
        int[] complexDimensionPartition = new int[complexDimColumnCardinality.length];
        // as complex dimension will be stored in column format add one
        Arrays.fill(complexDimensionPartition, 1);
        bitLength = CarbonUtil.getDimensionBitLength(complexDimColumnCardinality, complexDimensionPartition);
        for (int i = 0; i < bitLength.length; i++) {
            if (complexDimColumnCardinality[i] == 0) {
                bitLength[i] = 64;
            }
        }
        ColumnarSplitter keySplitter = new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, complexDimensionPartition);
        eachComplexDimColumnValueSize = keySplitter.getBlockKeySize();
    } else {
        eachComplexDimColumnValueSize = new int[0];
    }
}