Examples with MultiDimKeyVarLengthGenerator org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator used on opensource projects

Example 1 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class SegmentProperties method fillColumnGroupAndItsCardinality.

/**
 * Below method will be used to create a mapping of column group and its column cardinality this
 * mapping will have column group id to cardinality of the dimension present in
 * the column group.This mapping will be used during query execution, to create
 * a mask key for the column group dimension which will be used in aggregation
 * and filter query as column group dimension will be stored at the bit level
 */
private void fillColumnGroupAndItsCardinality(int[] cardinality) {
    // mapping of the column group and its ordinal
    Map<Integer, List<Integer>> columnGroupAndOrdinalMapping = new HashMap<Integer, List<Integer>>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    // to store a column group
    List<Integer> currentColumnGroup = null;
    // current index
    int index = 0;
    // previous column group to check all the column of column id has bee selected
    int prvColumnGroupId = -1;
    while (index < dimensions.size()) {
        // column group
        if (!dimensions.get(index).isColumnar() && dimensions.get(index).columnGroupId() == prvColumnGroupId && null != currentColumnGroup) {
            currentColumnGroup.add(index);
        } else // ordinal
        if (!dimensions.get(index).isColumnar()) {
            currentColumnGroup = new ArrayList<Integer>();
            columnGroupAndOrdinalMapping.put(dimensions.get(index).columnGroupId(), currentColumnGroup);
            currentColumnGroup.add(index);
        }
        // update the column id every time,this is required to group the
        // columns
        // of the same column group
        prvColumnGroupId = dimensions.get(index).columnGroupId();
        index++;
    }
    // Initializing the map
    this.columnGroupAndItsKeygenartor = new HashMap<Integer, KeyGenerator>(columnGroupAndOrdinalMapping.size());
    this.columnGroupOrdinalToMdkeymapping = new HashMap<>(columnGroupAndOrdinalMapping.size());
    int[] columnGroupCardinality = null;
    index = 0;
    Iterator<Entry<Integer, List<Integer>>> iterator = columnGroupAndOrdinalMapping.entrySet().iterator();
    while (iterator.hasNext()) {
        Entry<Integer, List<Integer>> next = iterator.next();
        List<Integer> currentGroupOrdinal = next.getValue();
        Map<Integer, Integer> colGrpOrdinalMdkeyMapping = new HashMap<>(currentGroupOrdinal.size());
        // create the cardinality array
        columnGroupCardinality = new int[currentGroupOrdinal.size()];
        for (int i = 0; i < columnGroupCardinality.length; i++) {
            // fill the cardinality
            columnGroupCardinality[i] = cardinality[currentGroupOrdinal.get(i)];
            colGrpOrdinalMdkeyMapping.put(currentGroupOrdinal.get(i), i);
        }
        this.columnGroupAndItsKeygenartor.put(next.getKey(), new MultiDimKeyVarLengthGenerator(CarbonUtil.getDimensionBitLength(columnGroupCardinality, new int[] { columnGroupCardinality.length })));
        this.columnGroupOrdinalToMdkeymapping.put(next.getKey(), colGrpOrdinalMdkeyMapping);
    }
}

Example 2 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2() throws KeyGenException {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
    int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
    KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    IndexKey key = new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 0, 0 }), null);
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(0, findFirstBlock.nodeIndex());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(0, findLastBlock.nodeIndex());
}

Example 3 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class BTreeBlockFinderTest method getDataFileFooterList.

private List<DataFileFooter> getDataFileFooterList() {
    List<DataFileFooter> list = new ArrayList<DataFileFooter>();
    try {
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
        KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
        int i = 1;
        while (i < 1001) {
            byte[] startKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i, i });
            byte[] endKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i + 10, i + 10 });
            ByteBuffer buffer = ByteBuffer.allocate(4 + 1);
            buffer.rewind();
            buffer.put((byte) 1);
            buffer.putInt(i);
            buffer.array();
            byte[] noDictionaryStartKey = buffer.array();
            ByteBuffer buffer1 = ByteBuffer.allocate(4 + 1);
            buffer1.rewind();
            buffer1.put((byte) 1);
            buffer1.putInt(i + 10);
            buffer1.array();
            byte[] noDictionaryEndKey = buffer1.array();
            DataFileFooter footer = getFileFooter(startKey, endKey, noDictionaryStartKey, noDictionaryEndKey);
            list.add(footer);
            i = i + 10;
        }
    } catch (Exception e) {
        LOGGER.error(e);
    }
    return list;
}

Example 4 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey.

/**
 * Below method will test when key which is not present and key which is
 * more than
 * last node key is passes for searching it should give first block
 */
@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey() throws KeyGenException {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
    int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
    KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    IndexKey key = new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 10001, 10001 }), null);
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(99, findFirstBlock.nodeIndex());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(99, findLastBlock.nodeIndex());
}

Example 5 with MultiDimKeyVarLengthGenerator

use of org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey1.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey1() throws KeyGenException {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
    int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
    KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    IndexKey key = new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 1, 1 }), null);
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(0, findFirstBlock.nodeIndex());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(0, findLastBlock.nodeIndex());
}