Examples with IndexKey org.apache.carbondata.core.datastore.IndexKey used on opensource projects

Example 11 with IndexKey

use of org.apache.carbondata.core.datastore.IndexKey in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1() {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyNoDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
    ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
    buffer.rewind();
    buffer.putShort((short) 1);
    buffer.putInt(12);
    buffer.array();
    IndexKey key = new IndexKey(null, buffer.array());
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(1, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(1, findLastBlock.nodeNumber());
}

Example 12 with IndexKey

use of org.apache.carbondata.core.datastore.IndexKey in project carbondata by apache.

the class BlockLevelTraverser method getBlockRowMapping.

/**
   *
   * @param abstractIndex
   * @param blockRowMap
   * @param segId
   * @param updateStatusManager
   * @throws KeyGenException
   */
public long getBlockRowMapping(AbstractIndex abstractIndex, Map<String, Long> blockRowMap, String segId, SegmentUpdateStatusManager updateStatusManager) throws KeyGenException {
    IndexKey searchStartKey = FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties());
    DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(abstractIndex.getSegmentProperties().getEachDimColumnValueSize(), abstractIndex.getSegmentProperties().getNumberOfSortColumns(), abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns());
    DataRefNode currentBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey);
    long count = 0;
    while (currentBlock != null) {
        String blockName = ((BlockBTreeLeafNode) currentBlock).getTableBlockInfo().getFilePath();
        blockName = CarbonTablePath.getCarbonDataFileName(blockName);
        blockName = blockName + CarbonTablePath.getCarbonDataExtension();
        long rowCount = currentBlock.nodeSize();
        String key = CarbonUpdateUtil.getSegmentBlockNameKey(segId, blockName);
        // if block is invalid then dont add the count
        SegmentUpdateDetails details = updateStatusManager.getDetailsForABlock(key);
        if (null == details || !CarbonUpdateUtil.isBlockInvalid(details.getStatus())) {
            blockRowMap.put(key, rowCount);
            count++;
        }
        currentBlock = currentBlock.getNextDataRefNode();
    }
    return count;
}

Example 13 with IndexKey

use of org.apache.carbondata.core.datastore.IndexKey in project carbondata by apache.

the class FilterUtilTest method testPrepareDefaultStartIndexKey.

@Test
public void testPrepareDefaultStartIndexKey() throws KeyGenException {
    List<ColumnSchema> columnsInTable = new ArrayList<>();
    columnsInTable.add(columnSchema);
    int[] columnCardinality = new int[] { 1, 2 };
    new MockUp<ColumnSchema>() {

        @Mock
        public List<Encoding> getEncodingList() {
            List<Encoding> encodingList = new ArrayList<>();
            encodingList.add(Encoding.DICTIONARY);
            return encodingList;
        }
    };
    SegmentProperties segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
    assertTrue(FilterUtil.prepareDefaultStartIndexKey(segmentProperties) instanceof IndexKey);
}

Example 14 with IndexKey

use of org.apache.carbondata.core.datastore.IndexKey in project carbondata by apache.

the class FilterUtil method traverseResolverTreeAndGetStartAndEndKey.

/**
   * method will set the start and end key for as per the filter resolver tree
   * utilized visitor pattern inorder to populate the start and end key population.
   *
   * @param segmentProperties
   * @param filterResolver
   * @param listOfStartEndKeys
   */
public static void traverseResolverTreeAndGetStartAndEndKey(SegmentProperties segmentProperties, FilterResolverIntf filterResolver, List<IndexKey> listOfStartEndKeys) {
    IndexKey searchStartKey = null;
    IndexKey searchEndKey = null;
    long[] startKey = new long[segmentProperties.getNumberOfDictSortColumns()];
    long[] endKey = new long[segmentProperties.getNumberOfDictSortColumns()];
    List<byte[]> listOfStartKeyByteArray = new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
    List<byte[]> listOfEndKeyByteArray = new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
    SortedMap<Integer, byte[]> setOfStartKeyByteArray = new TreeMap<Integer, byte[]>();
    SortedMap<Integer, byte[]> setOfEndKeyByteArray = new TreeMap<Integer, byte[]>();
    SortedMap<Integer, byte[]> defaultStartValues = new TreeMap<Integer, byte[]>();
    SortedMap<Integer, byte[]> defaultEndValues = new TreeMap<Integer, byte[]>();
    List<long[]> startKeyList = new ArrayList<long[]>();
    List<long[]> endKeyList = new ArrayList<long[]>();
    traverseResolverTreeAndPopulateStartAndEndKeys(filterResolver, segmentProperties, startKey, setOfStartKeyByteArray, endKey, setOfEndKeyByteArray, startKeyList, endKeyList);
    if (endKeyList.size() > 0) {
        //get the new end key from list
        for (int i = 0; i < endKey.length; i++) {
            long[] endkeyColumnLevel = new long[endKeyList.size()];
            int j = 0;
            for (long[] oneEndKey : endKeyList) {
                //get each column level end key
                endkeyColumnLevel[j++] = oneEndKey[i];
            }
            Arrays.sort(endkeyColumnLevel);
            // get the max one as end of this column level
            endKey[i] = endkeyColumnLevel[endkeyColumnLevel.length - 1];
        }
    }
    if (startKeyList.size() > 0) {
        //get the new start key from list
        for (int i = 0; i < startKey.length; i++) {
            long[] startkeyColumnLevel = new long[startKeyList.size()];
            int j = 0;
            for (long[] oneStartKey : startKeyList) {
                //get each column level start key
                startkeyColumnLevel[j++] = oneStartKey[i];
            }
            Arrays.sort(startkeyColumnLevel);
            // get the min - 1 as start of this column level, for example if a block contains 5,6
            // the filter is 6, but that block's start key is 5, if not -1, this block will missing.
            startKey[i] = startkeyColumnLevel[0] - 1;
        }
    }
    fillDefaultStartValue(defaultStartValues, segmentProperties);
    fillDefaultEndValue(defaultEndValues, segmentProperties);
    fillNullValuesStartIndexWithDefaultKeys(setOfStartKeyByteArray, segmentProperties);
    fillNullValuesEndIndexWithDefaultKeys(setOfEndKeyByteArray, segmentProperties);
    pruneStartAndEndKeys(setOfStartKeyByteArray, listOfStartKeyByteArray);
    pruneStartAndEndKeys(setOfEndKeyByteArray, listOfEndKeyByteArray);
    if (segmentProperties.getNumberOfNoDictSortColumns() == 0) {
        listOfStartKeyByteArray = new ArrayList<byte[]>();
        listOfEndKeyByteArray = new ArrayList<byte[]>();
    } else if (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray.size()) {
        while (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray.size()) {
            listOfStartKeyByteArray.remove(listOfStartKeyByteArray.size() - 1);
            listOfEndKeyByteArray.remove(listOfEndKeyByteArray.size() - 1);
        }
    }
    searchStartKey = FilterUtil.createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getSortColumnsGenerator(), FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray));
    searchEndKey = FilterUtil.createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getSortColumnsGenerator(), FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray));
    listOfStartEndKeys.add(searchStartKey);
    listOfStartEndKeys.add(searchEndKey);
}

Example 15 with IndexKey

use of org.apache.carbondata.core.datastore.IndexKey in project carbondata by apache.

the class CarbonInputFormat method getDataBlocksOfIndex.

/**
   * get data blocks of given btree
   */
private List<DataRefNode> getDataBlocksOfIndex(AbstractIndex abstractIndex) {
    List<DataRefNode> blocks = new LinkedList<DataRefNode>();
    SegmentProperties segmentProperties = abstractIndex.getSegmentProperties();
    try {
        IndexKey startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
        IndexKey endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
        // Add all blocks of btree into result
        DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(), segmentProperties.getNumberOfSortColumns(), segmentProperties.getNumberOfNoDictSortColumns());
        DataRefNode startBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
        DataRefNode endBlock = blockFinder.findLastDataBlock(abstractIndex.getDataRefNode(), endIndexKey);
        while (startBlock != endBlock) {
            blocks.add(startBlock);
            startBlock = startBlock.getNextDataRefNode();
        }
        blocks.add(endBlock);
    } catch (KeyGenException e) {
        LOG.error("Could not generate start key", e);
    }
    return blocks;
}