Examples with LocalDictionaryGenerator org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator used on opensource projects

Example 6 with LocalDictionaryGenerator

use of org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator in project carbondata by apache.

the class TestLocalDictionaryGenerator method testColumnLocalDictionaryGeneratorWhenThresholdReached_ExceptionShouldBeThrown.

@Test
public void testColumnLocalDictionaryGeneratorWhenThresholdReached_ExceptionShouldBeThrown() {
    LocalDictionaryGenerator generator = new ColumnLocalDictionaryGenerator(1000, 2);
    try {
        for (int i = 1; i <= 10000; i++) {
            generator.generateDictionary(("" + i).getBytes());
        }
        Assert.assertTrue(false);
    } catch (DictionaryThresholdReachedException e) {
        Assert.assertTrue(true);
    }
    Assert.assertTrue(generator.isThresholdReached());
}

Example 7 with LocalDictionaryGenerator

use of org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator in project carbondata by apache.

the class TestLocalDictionaryGenerator method testColumnLocalDictionaryGeneratorWithValidDataWithinThreshold.

@Test
public void testColumnLocalDictionaryGeneratorWithValidDataWithinThreshold() {
    LocalDictionaryGenerator generator = new ColumnLocalDictionaryGenerator(1000, 2);
    try {
        for (int i = 1; i <= 1000; i++) {
            generator.generateDictionary(("" + i).getBytes());
        }
        Assert.assertTrue(true);
    } catch (Exception e) {
        Assert.assertTrue(false);
    }
    int dictionaryValue = 2;
    for (int i = 1; i <= 1000; i++) {
        byte[] dictionaryKeyBasedOnValue = generator.getDictionaryKeyBasedOnValue(dictionaryValue);
        Assert.assertTrue(Arrays.equals(dictionaryKeyBasedOnValue, ("" + i).getBytes()));
        dictionaryValue++;
    }
}

Example 8 with LocalDictionaryGenerator

use of org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator in project carbondata by apache.

the class CarbonUtil method getLocalDictionaryModel.

/**
 * This method prepares a map which will have column and local dictionary generator mapping for
 * all the local dictionary columns.
 *
 * @param carbonTable
 * carbon Table
 */
public static Map<String, LocalDictionaryGenerator> getLocalDictionaryModel(CarbonTable carbonTable) {
    List<ColumnSchema> wrapperColumnSchema = CarbonUtil.getColumnSchemaList(carbonTable.getVisibleDimensions(), carbonTable.getVisibleMeasures());
    boolean isLocalDictEnabled = carbonTable.isLocalDictionaryEnabled();
    // creates a map only if local dictionary is enabled, else map will be null
    Map<String, LocalDictionaryGenerator> columnLocalDictGenMap = new HashMap<>();
    if (isLocalDictEnabled) {
        int localDictionaryThreshold = carbonTable.getLocalDictionaryThreshold();
        for (ColumnSchema columnSchema : wrapperColumnSchema) {
            // check whether the column is local dictionary column or not
            if (columnSchema.isLocalDictColumn()) {
                columnLocalDictGenMap.put(columnSchema.getColumnName(), new ColumnLocalDictionaryGenerator(localDictionaryThreshold, columnSchema.getDataType() == DataTypes.VARCHAR ? CarbonCommonConstants.INT_SIZE_IN_BYTE : CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
            }
        }
    }
    if (isLocalDictEnabled) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Local dictionary is enabled for table: " + carbonTable.getTableUniqueName());
            LOGGER.debug(String.format("Local dictionary threshold for table %s is %d", carbonTable.getTableUniqueName(), carbonTable.getLocalDictionaryThreshold()));
        }
        Iterator<Map.Entry<String, LocalDictionaryGenerator>> iterator = columnLocalDictGenMap.entrySet().iterator();
        StringBuilder stringBuilder = new StringBuilder();
        while (iterator.hasNext()) {
            Map.Entry<String, LocalDictionaryGenerator> next = iterator.next();
            stringBuilder.append(next.getKey());
            stringBuilder.append(',');
        }
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(String.format("Local dictionary will be generated for the columns: %s for" + " table %s", stringBuilder.toString(), carbonTable.getTableUniqueName()));
        }
    }
    return columnLocalDictGenMap;
}