Examples with DictionaryByteArrayWrapper org.apache.carbondata.core.cache.dictionary.DictionaryByteArrayWrapper used on opensource projects

Example 1 with DictionaryByteArrayWrapper

use of org.apache.carbondata.core.cache.dictionary.DictionaryByteArrayWrapper in project carbondata by apache.

the class DictionaryByteArrayWrapperTest method equalsTestWithDifferentLength.

@Test
public void equalsTestWithDifferentLength() {
    Boolean res = dictionaryByteArrayWrapper.equals(new DictionaryByteArrayWrapper("Rahul ".getBytes()));
    Assert.assertTrue(!res);
}

Example 2 with DictionaryByteArrayWrapper

use of org.apache.carbondata.core.cache.dictionary.DictionaryByteArrayWrapper in project carbondata by apache.

the class DictionaryByteArrayWrapperTest method setup.

@BeforeClass
public static void setup() {
    byte[] data = "Rahul".getBytes();
    dictionaryByteArrayWrapper = new DictionaryByteArrayWrapper(data);
    dictionaryByteArrayWrapper1 = new DictionaryByteArrayWrapper(data, XXHashFactory.fastestInstance().hash32());
}

Example 3 with DictionaryByteArrayWrapper

use of org.apache.carbondata.core.cache.dictionary.DictionaryByteArrayWrapper in project carbondata by apache.

the class DictionaryByteArrayWrapperTest method equalsTestWithDictionaryByteArrayWrapper.

@Test
public void equalsTestWithDictionaryByteArrayWrapper() {
    Boolean res = dictionaryByteArrayWrapper.equals(new DictionaryByteArrayWrapper("Rahul".getBytes()));
    Assert.assertTrue(res);
}

Example 4 with DictionaryByteArrayWrapper

use of org.apache.carbondata.core.cache.dictionary.DictionaryByteArrayWrapper in project carbondata by apache.

the class MapBasedDictionaryStore method putIfAbsent.

/**
 * Below method will be used to add dictionary value to dictionary holder
 * if it is already present in the holder then it will return exiting dictionary value.
 *
 * @param data dictionary key
 * @return dictionary value
 */
@Override
public int putIfAbsent(byte[] data) throws DictionaryThresholdReachedException {
    // check if threshold has already reached
    checkIfThresholdReached();
    DictionaryByteArrayWrapper key = new DictionaryByteArrayWrapper(data);
    // get the dictionary value
    Integer value = dictionary.get(key);
    // if value is null then dictionary is not present in store
    if (null == value) {
        // acquire the lock
        synchronized (dictionary) {
            // check threshold
            checkIfThresholdReached();
            // get the value again as other thread might have added
            value = dictionary.get(key);
            // double checking
            if (null == value) {
                // increment the value
                value = ++lastAssignValue;
                currentSize += data.length;
                // if new value is greater than threshold
                if (value > dictionaryThreshold || currentSize > dictionarySizeThresholdInBytes) {
                    // set the threshold boolean to true
                    isThresholdReached = true;
                    // throw exception
                    checkIfThresholdReached();
                }
                // add to reference array
                // position is -1 as dictionary value starts from 1
                this.referenceDictionaryArray[value - 1] = key;
                dictionary.put(key, value);
            }
        }
    }
    return value;
}

Example 5 with DictionaryByteArrayWrapper

use of org.apache.carbondata.core.cache.dictionary.DictionaryByteArrayWrapper in project carbondata by apache.

the class CarbonTestUtil method validateDictionary.

public static Boolean validateDictionary(DimensionRawColumnChunk rawColumnPage, String[] data) throws IOException {
    LocalDictionaryChunk local_dictionary = rawColumnPage.getDataChunkV3().local_dictionary;
    if (null != local_dictionary) {
        String compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta(rawColumnPage.getDataChunkV3().getData_chunk_list().get(0).getChunk_meta());
        List<org.apache.carbondata.format.Encoding> encodings = local_dictionary.getDictionary_meta().encoders;
        DefaultEncodingFactory encodingFactory = (DefaultEncodingFactory) DefaultEncodingFactory.getInstance();
        ColumnPageDecoder decoder = encodingFactory.createDecoder(encodings, local_dictionary.getDictionary_meta().getEncoder_meta(), compressorName);
        LazyColumnPage dictionaryPage = (LazyColumnPage) decoder.decode(local_dictionary.getDictionary_data(), 0, local_dictionary.getDictionary_data().length);
        HashMap<DictionaryByteArrayWrapper, Integer> dictionaryMap = new HashMap<>();
        BitSet usedDictionaryValues = BitSet.valueOf(CompressorFactory.getInstance().getCompressor(compressorName).unCompressByte(local_dictionary.getDictionary_values()));
        int index = 0;
        int i = usedDictionaryValues.nextSetBit(0);
        while (i >= 0) {
            dictionaryMap.put(new DictionaryByteArrayWrapper(dictionaryPage.getBytes(index)), i);
            i = usedDictionaryValues.nextSetBit(i + 1);
            index += 1;
        }
        for (i = 0; i < data.length; i++) {
            if (null == dictionaryMap.get(new DictionaryByteArrayWrapper(data[i].getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET))))) {
                return false;
            }
        }
        return true;
    }
    return false;
}