Search in sources :

Example 6 with LocalDictionaryGenerator

use of org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator in project carbondata by apache.

the class TestLocalDictionaryGenerator method testColumnLocalDictionaryGeneratorWhenThresholdReached_ExceptionShouldBeThrown.

@Test
public void testColumnLocalDictionaryGeneratorWhenThresholdReached_ExceptionShouldBeThrown() {
    LocalDictionaryGenerator generator = new ColumnLocalDictionaryGenerator(1000, 2);
    try {
        for (int i = 1; i <= 10000; i++) {
            generator.generateDictionary(("" + i).getBytes());
        }
        Assert.assertTrue(false);
    } catch (DictionaryThresholdReachedException e) {
        Assert.assertTrue(true);
    }
    Assert.assertTrue(generator.isThresholdReached());
}
Also used : ColumnLocalDictionaryGenerator(org.apache.carbondata.core.localdictionary.generator.ColumnLocalDictionaryGenerator) LocalDictionaryGenerator(org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator) DictionaryThresholdReachedException(org.apache.carbondata.core.localdictionary.exception.DictionaryThresholdReachedException) ColumnLocalDictionaryGenerator(org.apache.carbondata.core.localdictionary.generator.ColumnLocalDictionaryGenerator) Test(org.junit.Test)

Example 7 with LocalDictionaryGenerator

use of org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator in project carbondata by apache.

the class TestLocalDictionaryGenerator method testColumnLocalDictionaryGeneratorWithValidDataWithinThreshold.

@Test
public void testColumnLocalDictionaryGeneratorWithValidDataWithinThreshold() {
    LocalDictionaryGenerator generator = new ColumnLocalDictionaryGenerator(1000, 2);
    try {
        for (int i = 1; i <= 1000; i++) {
            generator.generateDictionary(("" + i).getBytes());
        }
        Assert.assertTrue(true);
    } catch (Exception e) {
        Assert.assertTrue(false);
    }
    int dictionaryValue = 2;
    for (int i = 1; i <= 1000; i++) {
        byte[] dictionaryKeyBasedOnValue = generator.getDictionaryKeyBasedOnValue(dictionaryValue);
        Assert.assertTrue(Arrays.equals(dictionaryKeyBasedOnValue, ("" + i).getBytes()));
        dictionaryValue++;
    }
}
Also used : ColumnLocalDictionaryGenerator(org.apache.carbondata.core.localdictionary.generator.ColumnLocalDictionaryGenerator) LocalDictionaryGenerator(org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator) ColumnLocalDictionaryGenerator(org.apache.carbondata.core.localdictionary.generator.ColumnLocalDictionaryGenerator) DictionaryThresholdReachedException(org.apache.carbondata.core.localdictionary.exception.DictionaryThresholdReachedException) Test(org.junit.Test)

Example 8 with LocalDictionaryGenerator

use of org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator in project carbondata by apache.

the class CarbonUtil method getLocalDictionaryModel.

/**
 * This method prepares a map which will have column and local dictionary generator mapping for
 * all the local dictionary columns.
 *
 * @param carbonTable
 * carbon Table
 */
public static Map<String, LocalDictionaryGenerator> getLocalDictionaryModel(CarbonTable carbonTable) {
    List<ColumnSchema> wrapperColumnSchema = CarbonUtil.getColumnSchemaList(carbonTable.getVisibleDimensions(), carbonTable.getVisibleMeasures());
    boolean isLocalDictEnabled = carbonTable.isLocalDictionaryEnabled();
    // creates a map only if local dictionary is enabled, else map will be null
    Map<String, LocalDictionaryGenerator> columnLocalDictGenMap = new HashMap<>();
    if (isLocalDictEnabled) {
        int localDictionaryThreshold = carbonTable.getLocalDictionaryThreshold();
        for (ColumnSchema columnSchema : wrapperColumnSchema) {
            // check whether the column is local dictionary column or not
            if (columnSchema.isLocalDictColumn()) {
                columnLocalDictGenMap.put(columnSchema.getColumnName(), new ColumnLocalDictionaryGenerator(localDictionaryThreshold, columnSchema.getDataType() == DataTypes.VARCHAR ? CarbonCommonConstants.INT_SIZE_IN_BYTE : CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
            }
        }
    }
    if (isLocalDictEnabled) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Local dictionary is enabled for table: " + carbonTable.getTableUniqueName());
            LOGGER.debug(String.format("Local dictionary threshold for table %s is %d", carbonTable.getTableUniqueName(), carbonTable.getLocalDictionaryThreshold()));
        }
        Iterator<Map.Entry<String, LocalDictionaryGenerator>> iterator = columnLocalDictGenMap.entrySet().iterator();
        StringBuilder stringBuilder = new StringBuilder();
        while (iterator.hasNext()) {
            Map.Entry<String, LocalDictionaryGenerator> next = iterator.next();
            stringBuilder.append(next.getKey());
            stringBuilder.append(',');
        }
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(String.format("Local dictionary will be generated for the columns: %s for" + " table %s", stringBuilder.toString(), carbonTable.getTableUniqueName()));
        }
    }
    return columnLocalDictGenMap;
}
Also used : HashMap(java.util.HashMap) ColumnSchema(org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema) ColumnLocalDictionaryGenerator(org.apache.carbondata.core.localdictionary.generator.ColumnLocalDictionaryGenerator) SchemaEvolutionEntry(org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry) ColumnLocalDictionaryGenerator(org.apache.carbondata.core.localdictionary.generator.ColumnLocalDictionaryGenerator) LocalDictionaryGenerator(org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator) Map(java.util.Map) HashMap(java.util.HashMap)

Aggregations

LocalDictionaryGenerator (org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator)8 ColumnLocalDictionaryGenerator (org.apache.carbondata.core.localdictionary.generator.ColumnLocalDictionaryGenerator)7 Test (org.junit.Test)6 DictionaryThresholdReachedException (org.apache.carbondata.core.localdictionary.exception.DictionaryThresholdReachedException)5 ByteBuffer (java.nio.ByteBuffer)3 IOException (java.io.IOException)2 ColumnPage (org.apache.carbondata.core.datastore.page.ColumnPage)2 ColumnPageDecoder (org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder)2 DefaultEncodingFactory (org.apache.carbondata.core.datastore.page.encoding.DefaultEncodingFactory)2 EncodingFactory (org.apache.carbondata.core.datastore.page.encoding.EncodingFactory)2 Encoding (org.apache.carbondata.format.Encoding)2 LocalDictionaryChunk (org.apache.carbondata.format.LocalDictionaryChunk)2 BitSet (java.util.BitSet)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 TableSpec (org.apache.carbondata.core.datastore.TableSpec)1 ColumnPageEncoderMeta (org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta)1 DummyStatsCollector (org.apache.carbondata.core.datastore.page.statistics.DummyStatsCollector)1 DataType (org.apache.carbondata.core.metadata.datatype.DataType)1 SchemaEvolutionEntry (org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry)1