Examples with CacheProvider org.apache.carbondata.core.cache.CacheProvider used on opensource projects

Example 6 with CacheProvider

use of org.apache.carbondata.core.cache.CacheProvider in project carbondata by apache.

the class RowConverterImpl method initialize.

@Override
public void initialize() throws IOException {
    CacheProvider cacheProvider = CacheProvider.getInstance();
    cache = cacheProvider.createCache(CacheType.REVERSE_DICTIONARY, configuration.getTableIdentifier().getStorePath());
    String nullFormat = configuration.getDataLoadProperty(DataLoadProcessorConstants.SERIALIZATION_NULL_FORMAT).toString();
    boolean isEmptyBadRecord = Boolean.parseBoolean(configuration.getDataLoadProperty(DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD).toString());
    List<FieldConverter> fieldConverterList = new ArrayList<>();
    localCaches = new Map[fields.length];
    long lruCacheStartTime = System.currentTimeMillis();
    DictionaryClient client = createDictionaryClient();
    dictClients.add(client);
    for (int i = 0; i < fields.length; i++) {
        localCaches[i] = new ConcurrentHashMap<>();
        FieldConverter fieldConverter = FieldEncoderFactory.getInstance().createFieldEncoder(fields[i], cache, configuration.getTableIdentifier().getCarbonTableIdentifier(), i, nullFormat, client, configuration.getUseOnePass(), configuration.getTableIdentifier().getStorePath(), true, localCaches[i], isEmptyBadRecord);
        fieldConverterList.add(fieldConverter);
    }
    CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordLruCacheLoadTime((System.currentTimeMillis() - lruCacheStartTime) / 1000.0);
    fieldConverters = fieldConverterList.toArray(new FieldConverter[fieldConverterList.size()]);
    logHolder = new BadRecordLogHolder();
}

Example 7 with CacheProvider

use of org.apache.carbondata.core.cache.CacheProvider in project carbondata by apache.

the class SegmentTaskIndexStoreTest method setUp.

@BeforeClass
public static void setUp() {
    CacheProvider cacheProvider = CacheProvider.getInstance();
    taskIndexStore = (SegmentTaskIndexStore) cacheProvider.<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper>createCache(CacheType.DRIVER_BTREE, "");
    tableBlockInfo = new TableBlockInfo("file", 0L, "SG100", locations, 10L, ColumnarFormatVersion.valueOf(version));
    absoluteTableIdentifier = new AbsoluteTableIdentifier("/tmp", new CarbonTableIdentifier("testdatabase", "testtable", "TB100"));
}

Example 8 with CacheProvider

use of org.apache.carbondata.core.cache.CacheProvider in project carbondata by apache.

the class ForwardDictionaryCacheTest method createDictionaryCacheObject.

private void createDictionaryCacheObject() {
    // enable lru cache by setting cache size
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "10");
    CacheProvider cacheProvider = CacheProvider.getInstance();
    forwardDictionaryCache = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, this.carbonStorePath);
}

Example 9 with CacheProvider

use of org.apache.carbondata.core.cache.CacheProvider in project carbondata by apache.

the class DictionaryDecodeReadSupport method initialize.

/**
   * This initialization is done inside executor task
   * for column dictionary involved in decoding.
   *
   * @param carbonColumns column list
   * @param absoluteTableIdentifier table identifier
   */
@Override
public void initialize(CarbonColumn[] carbonColumns, AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
    this.carbonColumns = carbonColumns;
    dictionaries = new Dictionary[carbonColumns.length];
    dataTypes = new DataType[carbonColumns.length];
    for (int i = 0; i < carbonColumns.length; i++) {
        if (carbonColumns[i].hasEncoding(Encoding.DICTIONARY) && !carbonColumns[i].hasEncoding(Encoding.DIRECT_DICTIONARY) && !carbonColumns[i].isComplex()) {
            CacheProvider cacheProvider = CacheProvider.getInstance();
            Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath());
            dataTypes[i] = carbonColumns[i].getDataType();
            dictionaries[i] = forwardDictionaryCache.get(new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier.getCarbonTableIdentifier(), carbonColumns[i].getColumnIdentifier(), dataTypes[i]));
        } else {
            dataTypes[i] = carbonColumns[i].getDataType();
        }
    }
}