Examples with KeyGenerator org.apache.carbondata.core.keygenerator.KeyGenerator used on opensource projects

Example 11 with KeyGenerator

use of org.apache.carbondata.core.keygenerator.KeyGenerator in project carbondata by apache.

the class QueryUtil method getColumnGroupKeyStructureInfo.

/**
   * Below method will be used to get the mapping of block index and its
   * restructuring info
   *
   * @param queryDimensions   query dimension from query model
   * @param segmentProperties segment properties
   * @return map of block index to its restructuring info
   * @throws KeyGenException if problem while key generation
   */
public static Map<Integer, KeyStructureInfo> getColumnGroupKeyStructureInfo(List<QueryDimension> queryDimensions, SegmentProperties segmentProperties) throws KeyGenException {
    Map<Integer, KeyStructureInfo> rowGroupToItsRSInfo = new HashMap<Integer, KeyStructureInfo>();
    // get column group id and its ordinal mapping of column group
    Map<Integer, List<Integer>> columnGroupAndItsOrdinalMappingForQuery = getColumnGroupAndItsOrdinalMapping(queryDimensions);
    Map<Integer, KeyGenerator> columnGroupAndItsKeygenartor = segmentProperties.getColumnGroupAndItsKeygenartor();
    Iterator<Entry<Integer, List<Integer>>> iterator = columnGroupAndItsOrdinalMappingForQuery.entrySet().iterator();
    KeyStructureInfo restructureInfos = null;
    while (iterator.hasNext()) {
        Entry<Integer, List<Integer>> next = iterator.next();
        KeyGenerator keyGenerator = columnGroupAndItsKeygenartor.get(next.getKey());
        restructureInfos = new KeyStructureInfo();
        // sort the ordinal
        List<Integer> ordinal = next.getValue();
        List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
        //Un sorted
        List<Integer> mdKeyOrdinalForQuery = new ArrayList<Integer>();
        for (Integer ord : ordinal) {
            mdKeyOrdinal.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
            mdKeyOrdinalForQuery.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
        }
        Collections.sort(mdKeyOrdinal);
        // get the masked byte range for column group
        int[] maskByteRanges = getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
        // max key for column group
        byte[] maxKey = getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
        restructureInfos.setKeyGenerator(keyGenerator);
        restructureInfos.setMaskByteRanges(maskByteRanges);
        restructureInfos.setMaxKey(maxKey);
        restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils.toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()])));
        rowGroupToItsRSInfo.put(segmentProperties.getDimensionOrdinalToBlockMapping().get(ordinal.get(0)), restructureInfos);
    }
    return rowGroupToItsRSInfo;
}

Example 12 with KeyGenerator

use of org.apache.carbondata.core.keygenerator.KeyGenerator in project carbondata by apache.

the class QueryUtil method getKeyStructureInfo.

/**
   * Below method will be used to get the key structure for the column group
   *
   * @param segmentProperties      segment properties
   * @param dimColumnEvaluatorInfo dimension evaluator info
   * @return key structure info for column group dimension
   * @throws KeyGenException
   */
public static KeyStructureInfo getKeyStructureInfo(SegmentProperties segmentProperties, DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) throws KeyGenException {
    int colGrpId = getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
    KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
    List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
    mdKeyOrdinal.add(segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, dimColumnEvaluatorInfo.getColumnIndex()));
    int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
    byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
    KeyStructureInfo restructureInfos = new KeyStructureInfo();
    restructureInfos.setKeyGenerator(keyGenerator);
    restructureInfos.setMaskByteRanges(maskByteRanges);
    restructureInfos.setMaxKey(maxKey);
    return restructureInfos;
}

Example 13 with KeyGenerator

use of org.apache.carbondata.core.keygenerator.KeyGenerator in project carbondata by apache.

the class KeyGeneratorFactoryUnitTest method testGetKeyGenerato.

@Test
public void testGetKeyGenerato() throws Exception {
    int expected = 9;
    int[] dimCardinality = new int[] { 10, 20, 30, 11, 26, 52, 85, 65, 12 };
    int[] columnSplits = new int[] { 2 };
    KeyGenerator result = getKeyGenerator(dimCardinality, columnSplits);
    assertEquals(expected, result.getDimCount());
}

Example 14 with KeyGenerator

use of org.apache.carbondata.core.keygenerator.KeyGenerator in project carbondata by apache.

the class BTreeBlockFinderTest method getFileFooterListWithOnlyNoDictionaryKey.

private List<DataFileFooter> getFileFooterListWithOnlyNoDictionaryKey() {
    List<DataFileFooter> list = new ArrayList<DataFileFooter>();
    try {
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
        KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
        int i = 1;
        while (i < 1001) {
            byte[] startKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i, i });
            byte[] endKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i + 10, i + 10 });
            ByteBuffer buffer = ByteBuffer.allocate(2 + 4);
            buffer.rewind();
            buffer.putShort((short) 1);
            buffer.putInt(i);
            buffer.array();
            byte[] noDictionaryStartKey = buffer.array();
            ByteBuffer buffer1 = ByteBuffer.allocate(2 + 4);
            buffer1.rewind();
            buffer1.putShort((short) 2);
            buffer1.putInt(i + 10);
            buffer1.array();
            byte[] noDictionaryEndKey = buffer.array();
            DataFileFooter footer = getFileMatadataWithOnlyNoDictionaryKey(startKey, endKey, noDictionaryStartKey, noDictionaryEndKey);
            list.add(footer);
            i = i + 10;
        }
    } catch (Exception e) {
        LOGGER.error(e);
    }
    return list;
}

Example 15 with KeyGenerator

use of org.apache.carbondata.core.keygenerator.KeyGenerator in project carbondata by apache.

the class IncludeColGroupFilterExecuterImpl method getKeyStructureInfo.

/**
   * It is required for extracting column data from columngroup chunk
   *
   * @return
   * @throws KeyGenException
   */
private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
    int colGrpId = getColumnGroupId(dimColumnEvaluatorInfo.getColumnIndex());
    KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
    List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
    mdKeyOrdinal.add(getMdkeyOrdinal(dimColumnEvaluatorInfo.getColumnIndex(), colGrpId));
    int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
    byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
    KeyStructureInfo restructureInfos = new KeyStructureInfo();
    restructureInfos.setKeyGenerator(keyGenerator);
    restructureInfos.setMaskByteRanges(maskByteRanges);
    restructureInfos.setMaxKey(maxKey);
    return restructureInfos;
}