Examples with ColumnIdentifier org.apache.carbondata.core.metadata.ColumnIdentifier used on opensource projects

Example 1 with ColumnIdentifier

use of org.apache.carbondata.core.metadata.ColumnIdentifier in project carbondata by apache.

the class IncrementalColumnDictionaryGenerator method writeDictionaryData.

@Override
public void writeDictionaryData(String tableUniqueName) throws IOException {
    // initialize params
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    CarbonTable carbonTable = metadata.getCarbonTable(tableUniqueName);
    CarbonTableIdentifier tableIdentifier = carbonTable.getCarbonTableIdentifier();
    ColumnIdentifier columnIdentifier = dimension.getColumnIdentifier();
    String storePath = carbonTable.getStorePath();
    DictionaryService dictionaryService = CarbonCommonFactory.getDictionaryService();
    // create dictionary cache from dictionary File
    DictionaryColumnUniqueIdentifier identifier = new DictionaryColumnUniqueIdentifier(tableIdentifier, columnIdentifier, columnIdentifier.getDataType());
    Boolean isDictExists = CarbonUtil.isFileExistsForGivenColumn(storePath, identifier);
    Dictionary dictionary = null;
    long t1 = System.currentTimeMillis();
    if (isDictExists) {
        Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, storePath);
        dictionary = dictCache.get(identifier);
    }
    long dictCacheTime = System.currentTimeMillis() - t1;
    long t2 = System.currentTimeMillis();
    // write dictionary
    CarbonDictionaryWriter dictionaryWriter = null;
    dictionaryWriter = dictionaryService.getDictionaryWriter(tableIdentifier, columnIdentifier, storePath);
    List<String> distinctValues = writeDictionary(dictionaryWriter, isDictExists);
    long dictWriteTime = System.currentTimeMillis() - t2;
    long t3 = System.currentTimeMillis();
    // write sort index
    if (distinctValues.size() > 0) {
        writeSortIndex(distinctValues, dictionary, dictionaryService, tableIdentifier, columnIdentifier, storePath);
    }
    long sortIndexWriteTime = System.currentTimeMillis() - t3;
    // update Meta Data
    updateMetaData(dictionaryWriter);
    LOGGER.audit("\n columnName: " + dimension.getColName() + "\n columnId: " + dimension.getColumnId() + "\n new distinct values count: " + distinctValues.size() + "\n create dictionary cache: " + dictCacheTime + "\n sort list, distinct and write: " + dictWriteTime + "\n write sort info: " + sortIndexWriteTime);
}

Example 2 with ColumnIdentifier

use of org.apache.carbondata.core.metadata.ColumnIdentifier in project carbondata by apache.

the class QueryUtil method getDictionaryColumnUniqueIdentifierList.

/**
 * Below method will be used to get the dictionary column unique identifier
 *
 * @param dictionaryColumnIdList dictionary
 * @param carbonTableIdentifier
 * @return
 */
private static List<DictionaryColumnUniqueIdentifier> getDictionaryColumnUniqueIdentifierList(List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier, TableProvider tableProvider) throws IOException {
    CarbonTable carbonTable = tableProvider.getCarbonTable(carbonTableIdentifier);
    List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers = new ArrayList<>(dictionaryColumnIdList.size());
    for (String columnId : dictionaryColumnIdList) {
        CarbonDimension dimension = CarbonMetadata.getInstance().getCarbonDimensionBasedOnColIdentifier(carbonTable, columnId);
        if (dimension != null) {
            AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier;
            ColumnIdentifier columnIdentifier;
            if (null != dimension.getColumnSchema().getParentColumnTableRelations() && !dimension.getColumnSchema().getParentColumnTableRelations().isEmpty()) {
                dictionarySourceAbsoluteTableIdentifier = getTableIdentifierForColumn(dimension, carbonTable.getAbsoluteTableIdentifier());
                columnIdentifier = new ColumnIdentifier(dimension.getColumnSchema().getParentColumnTableRelations().get(0).getColumnId(), dimension.getColumnProperties(), dimension.getDataType());
            } else {
                dictionarySourceAbsoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
                columnIdentifier = dimension.getColumnIdentifier();
            }
            String dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties().get(CarbonCommonConstants.DICTIONARY_PATH);
            dictionaryColumnUniqueIdentifiers.add(new DictionaryColumnUniqueIdentifier(dictionarySourceAbsoluteTableIdentifier, columnIdentifier, dimension.getDataType(), dictionaryPath));
        }
    }
    return dictionaryColumnUniqueIdentifiers;
}

Example 3 with ColumnIdentifier

use of org.apache.carbondata.core.metadata.ColumnIdentifier in project carbondata by apache.

the class FilterUtil method getForwardDictionaryCache.

/**
 * @param carbonDimension
 * @param tableProvider
 * @return
 */
public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier, CarbonDimension carbonDimension, TableProvider tableProvider) throws IOException {
    String dictionaryPath = null;
    ColumnIdentifier columnIdentifier = carbonDimension.getColumnIdentifier();
    if (null != tableProvider) {
        CarbonTable carbonTable = tableProvider.getCarbonTable(dictionarySourceAbsoluteTableIdentifier.getCarbonTableIdentifier());
        dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties().get(CarbonCommonConstants.DICTIONARY_PATH);
        if (null != carbonDimension.getColumnSchema().getParentColumnTableRelations() && carbonDimension.getColumnSchema().getParentColumnTableRelations().size() == 1) {
            dictionarySourceAbsoluteTableIdentifier = QueryUtil.getTableIdentifierForColumn(carbonDimension, carbonTable.getAbsoluteTableIdentifier());
            columnIdentifier = new ColumnIdentifier(carbonDimension.getColumnSchema().getParentColumnTableRelations().get(0).getColumnId(), carbonDimension.getColumnProperties(), carbonDimension.getDataType());
        } else {
            dictionarySourceAbsoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
        }
    }
    DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(dictionarySourceAbsoluteTableIdentifier, columnIdentifier, carbonDimension.getDataType(), dictionaryPath);
    CacheProvider cacheProvider = CacheProvider.getInstance();
    Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY);
    // get the forward dictionary object
    return forwardDictionaryCache.get(dictionaryColumnUniqueIdentifier);
}

Example 4 with ColumnIdentifier

use of org.apache.carbondata.core.metadata.ColumnIdentifier in project carbondata by apache.

the class CarbonDictionaryWriterImplTest method setUp.

@Before
public void setUp() throws Exception {
    init();
    this.databaseName = props.getProperty("database", "testSchema");
    this.tableName = props.getProperty("tableName", "carbon");
    this.tablePath = props.getProperty("storePath", "carbonStore");
    this.columnIdentifier = new ColumnIdentifier("Name", null, null);
    carbonTableIdentifier = new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
    absoluteTableIdentifier = AbsoluteTableIdentifier.from(tablePath, carbonTableIdentifier);
    this.dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, columnIdentifier.getDataType());
    deleteStorePath();
    prepareDataSet();
}

Example 5 with ColumnIdentifier

use of org.apache.carbondata.core.metadata.ColumnIdentifier in project carbondata by apache.

the class CarbonDictionarySortIndexWriterImplTest method setUp.

@Before
public void setUp() throws Exception {
    storePath = "target/carbonStore";
    carbonTableIdentifier = new CarbonTableIdentifier("testSchema", "carbon", UUID.randomUUID().toString());
    String tablePath = storePath + "/" + carbonTableIdentifier.getDatabaseName() + "/" + carbonTableIdentifier.getTableName();
    absoluteTableIdentifier = AbsoluteTableIdentifier.from(tablePath, carbonTableIdentifier);
    columnIdentifier = new ColumnIdentifier("Name", null, null);
    DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, columnIdentifier.getDataType());
    dictionaryWriter = new CarbonDictionaryWriterImpl(dictionaryColumnUniqueIdentifier);
    dictionarySortIndexWriter = new CarbonDictionarySortIndexWriterImpl(dictionaryColumnUniqueIdentifier);
    carbonDictionarySortIndexReader = new CarbonDictionarySortIndexReaderImpl(dictionaryColumnUniqueIdentifier);
}