Examples with DictionaryChunksWrapper org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper used on opensource projects

Example 1 with DictionaryChunksWrapper

use of org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper in project carbondata by apache.

the class FilterUtil method prepareExcludeFilterMembers.

private static List<Integer> prepareExcludeFilterMembers(Dictionary forwardDictionary, List<Integer> includeSurrogates) throws FilterUnsupportedException {
    DictionaryChunksWrapper dictionaryWrapper;
    RoaringBitmap bitMapOfSurrogates = RoaringBitmap.bitmapOf(ArrayUtils.toPrimitive(includeSurrogates.toArray(new Integer[includeSurrogates.size()])));
    dictionaryWrapper = forwardDictionary.getDictionaryChunks();
    List<Integer> excludeFilterList = new ArrayList<Integer>(includeSurrogates.size());
    int surrogateCount = 0;
    while (dictionaryWrapper.hasNext()) {
        dictionaryWrapper.next();
        ++surrogateCount;
        if (!bitMapOfSurrogates.contains(surrogateCount)) {
            excludeFilterList.add(surrogateCount);
        }
    }
    return excludeFilterList;
}

Example 2 with DictionaryChunksWrapper

use of org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper in project carbondata by apache.

the class FilterUtil method getFilterListForAllValues.

/**
   * This method will get all the members of column from the forward dictionary
   * cache, this method will be basically used in row level filter resolver.
   *
   * @param tableIdentifier
   * @param expression
   * @param columnExpression
   * @param isIncludeFilter
   * @return DimColumnFilterInfo
   * @throws FilterUnsupportedException
   * @throws IOException
   */
public static DimColumnFilterInfo getFilterListForAllValues(AbsoluteTableIdentifier tableIdentifier, Expression expression, final ColumnExpression columnExpression, boolean isIncludeFilter) throws IOException, FilterUnsupportedException {
    Dictionary forwardDictionary = null;
    List<String> evaluateResultListFinal = new ArrayList<String>(20);
    DictionaryChunksWrapper dictionaryWrapper = null;
    try {
        forwardDictionary = getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension());
        dictionaryWrapper = forwardDictionary.getDictionaryChunks();
        while (dictionaryWrapper.hasNext()) {
            byte[] columnVal = dictionaryWrapper.next();
            try {
                RowIntf row = new RowImpl();
                String stringValue = new String(columnVal, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
                if (stringValue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
                    stringValue = null;
                }
                row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue, columnExpression.getCarbonColumn().getDataType()) });
                Boolean rslt = expression.evaluate(row).getBoolean();
                if (null != rslt && rslt == isIncludeFilter) {
                    if (null == stringValue) {
                        evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
                    } else {
                        evaluateResultListFinal.add(stringValue);
                    }
                }
            } catch (FilterIllegalMemberException e) {
                LOGGER.debug(e.getMessage());
            }
        }
        return getFilterValues(columnExpression, evaluateResultListFinal, forwardDictionary, isIncludeFilter);
    } finally {
        CarbonUtil.clearDictionaryCache(forwardDictionary);
    }
}

Example 3 with DictionaryChunksWrapper

use of org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper in project carbondata by apache.

the class CarbonDictionarySortInfoPreparator method prepareDictionarySortModels.

/**
 * The method returns the array of CarbonDictionarySortModel
 *
 * @param distinctValues new distinct values
 * @param dictionary The wrapper wraps the list<list<bye[]>> and provide the
 *                   iterator to retrieve the chunks members.
 * @param dataType   DataType of columns
 * @return CarbonDictionarySortModel[] CarbonDictionarySortModel[] the model
 * CarbonDictionarySortModel contains the  member's surrogate and
 * its byte value
 */
private CarbonDictionarySortModel[] prepareDictionarySortModels(List<String> distinctValues, Dictionary dictionary, DataType dataType) {
    CarbonDictionarySortModel[] dictionarySortModels = null;
    // The wrapper wraps the list<list<bye[]>> and provide the iterator to
    // retrieve the chunks members.
    int surrogate = 1;
    if (null != dictionary) {
        DictionaryChunksWrapper dictionaryChunksWrapper = dictionary.getDictionaryChunks();
        dictionarySortModels = new CarbonDictionarySortModel[dictionaryChunksWrapper.getSize() + distinctValues.size()];
        while (dictionaryChunksWrapper.hasNext()) {
            dictionarySortModels[surrogate - 1] = createDictionarySortModel(surrogate, dataType, dictionaryChunksWrapper.next());
            surrogate++;
        }
    } else {
        dictionarySortModels = new CarbonDictionarySortModel[distinctValues.size()];
    }
    // for new distinct values
    Iterator<String> distinctValue = distinctValues.iterator();
    while (distinctValue.hasNext()) {
        dictionarySortModels[surrogate - 1] = createDictionarySortModel(surrogate, dataType, distinctValue.next().getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
        surrogate++;
    }
    return dictionarySortModels;
}

Example 4 with DictionaryChunksWrapper

use of org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper in project carbondata by apache.

the class FilterUtil method prepareIncludeFilterMembers.

private static void prepareIncludeFilterMembers(Expression expression, final ColumnExpression columnExpression, boolean isIncludeFilter, Dictionary forwardDictionary, List<Integer> surrogates) throws FilterUnsupportedException {
    DictionaryChunksWrapper dictionaryWrapper;
    dictionaryWrapper = forwardDictionary.getDictionaryChunks();
    int surrogateCount = 0;
    while (dictionaryWrapper.hasNext()) {
        byte[] columnVal = dictionaryWrapper.next();
        ++surrogateCount;
        try {
            RowIntf row = new RowImpl();
            String stringValue = new String(columnVal, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
            if (stringValue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
                stringValue = null;
            }
            row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue, columnExpression.getCarbonColumn().getDataType()) });
            Boolean rslt = expression.evaluate(row).getBoolean();
            if (null != rslt) {
                if (rslt) {
                    if (null == stringValue) {
                        // this is for query like select name from table unknowexpr(name,1)
                        // != 'value' -> for null dictionary value
                        surrogates.add(CarbonCommonConstants.DICT_VALUE_NULL);
                    } else if (isIncludeFilter) {
                        // this is for query like select ** from * where unknwonexpr(*) == 'value'
                        surrogates.add(surrogateCount);
                    }
                } else if (null != stringValue && !isIncludeFilter) {
                    // this is for isNot null or not in query( e.x select ** from t where name is not null
                    surrogates.add(surrogateCount);
                }
            }
        } catch (FilterIllegalMemberException e) {
            LOGGER.debug(e.getMessage());
        }
    }
}

Example 5 with DictionaryChunksWrapper

use of org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper in project carbondata by apache.

the class CarbonDictionarySortInfoPreparatorTest method testGetDictionarySortInfo.

/**
 * Tests the getDictionarySortInfo method
 */
@Test
public void testGetDictionarySortInfo() {
    List<String> newDistinctValues = new ArrayList<>();
    newDistinctValues.add("abc");
    newDistinctValues.add("xyz");
    Dictionary dictionary = new MockUp<Dictionary>() {

        @Mock
        public DictionaryChunksWrapper getDictionaryChunks() {
            List<byte[]> data = new ArrayList<>();
            data.add(new byte[] { 1, 2 });
            List<List<byte[]>> dictionaryChunks = new ArrayList<>();
            dictionaryChunks.add(data);
            return new DictionaryChunksWrapper(dictionaryChunks);
        }
    }.getMockInstance();
    new MockUp<DictionaryChunksWrapper>() {

        @Mock
        public int getSize() {
            return 1;
        }
    };
    CarbonDictionarySortInfo carbonDictionarySortInfo = carbonDictionarySortInfoPreparator.getDictionarySortInfo(newDistinctValues, dictionary, DataTypes.STRING);
    int expectedGetSortIndexValue = 1;
    int expectedGetSortInvertedIndexLength = 3;
    int actualGetSortIndexValue = carbonDictionarySortInfo.getSortIndex().get(0);
    int actualGetSortInvertedIndexLength = carbonDictionarySortInfo.getSortIndexInverted().size();
    assertEquals(actualGetSortIndexValue, expectedGetSortIndexValue);
    assertEquals(actualGetSortInvertedIndexLength, expectedGetSortInvertedIndexLength);
}