Examples with CarbonMetadata org.apache.carbondata.core.metadata.CarbonMetadata used on opensource projects

Example 1 with CarbonMetadata

use of org.apache.carbondata.core.metadata.CarbonMetadata in project carbondata by apache.

the class IncrementalColumnDictionaryGenerator method writeDictionaryData.

@Override
public void writeDictionaryData(String tableUniqueName) throws IOException {
    // initialize params
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    CarbonTable carbonTable = metadata.getCarbonTable(tableUniqueName);
    CarbonTableIdentifier tableIdentifier = carbonTable.getCarbonTableIdentifier();
    ColumnIdentifier columnIdentifier = dimension.getColumnIdentifier();
    String storePath = carbonTable.getStorePath();
    DictionaryService dictionaryService = CarbonCommonFactory.getDictionaryService();
    // create dictionary cache from dictionary File
    DictionaryColumnUniqueIdentifier identifier = new DictionaryColumnUniqueIdentifier(tableIdentifier, columnIdentifier, columnIdentifier.getDataType());
    Boolean isDictExists = CarbonUtil.isFileExistsForGivenColumn(storePath, identifier);
    Dictionary dictionary = null;
    long t1 = System.currentTimeMillis();
    if (isDictExists) {
        Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, storePath);
        dictionary = dictCache.get(identifier);
    }
    long dictCacheTime = System.currentTimeMillis() - t1;
    long t2 = System.currentTimeMillis();
    // write dictionary
    CarbonDictionaryWriter dictionaryWriter = null;
    dictionaryWriter = dictionaryService.getDictionaryWriter(tableIdentifier, columnIdentifier, storePath);
    List<String> distinctValues = writeDictionary(dictionaryWriter, isDictExists);
    long dictWriteTime = System.currentTimeMillis() - t2;
    long t3 = System.currentTimeMillis();
    // write sort index
    if (distinctValues.size() > 0) {
        writeSortIndex(distinctValues, dictionary, dictionaryService, tableIdentifier, columnIdentifier, storePath);
    }
    long sortIndexWriteTime = System.currentTimeMillis() - t3;
    // update Meta Data
    updateMetaData(dictionaryWriter);
    LOGGER.audit("\n columnName: " + dimension.getColName() + "\n columnId: " + dimension.getColumnId() + "\n new distinct values count: " + distinctValues.size() + "\n create dictionary cache: " + dictCacheTime + "\n sort list, distinct and write: " + dictWriteTime + "\n write sort info: " + sortIndexWriteTime);
}

Example 2 with CarbonMetadata

use of org.apache.carbondata.core.metadata.CarbonMetadata in project carbondata by apache.

the class ServerDictionaryGenerator method initializeGeneratorForTable.

public void initializeGeneratorForTable(DictionaryMessage key) {
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    CarbonTable carbonTable = metadata.getCarbonTable(key.getTableUniqueName());
    CarbonDimension dimension = carbonTable.getPrimitiveDimensionByName(key.getTableUniqueName(), key.getColumnName());
    // initialize TableDictionaryGenerator first
    if (tableMap.get(key.getTableUniqueName()) == null) {
        tableMap.put(key.getTableUniqueName(), new TableDictionaryGenerator(dimension));
    } else {
        tableMap.get(key.getTableUniqueName()).updateGenerator(dimension);
    }
}

Example 3 with CarbonMetadata

use of org.apache.carbondata.core.metadata.CarbonMetadata in project carbondata by apache.

the class LuceneDataMapFactoryBase method init.

@Override
public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) throws IOException, MalformedDataMapCommandException {
    Objects.requireNonNull(identifier);
    Objects.requireNonNull(dataMapSchema);
    this.tableIdentifier = identifier;
    this.dataMapName = dataMapSchema.getDataMapName();
    // get carbonmetadata from carbonmetadata instance
    CarbonMetadata carbonMetadata = CarbonMetadata.getInstance();
    String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName();
    // get carbon table
    CarbonTable carbonTable = carbonMetadata.getCarbonTable(tableUniqueName);
    if (carbonTable == null) {
        String errorMessage = String.format("failed to get carbon table with name %s", tableUniqueName);
        LOGGER.error(errorMessage);
        throw new IOException(errorMessage);
    }
    // validate DataMapSchema and get index columns
    List<String> indexedColumns = validateAndGetIndexedColumns(dataMapSchema, carbonTable);
    // add optimizedOperations
    List<ExpressionType> optimizedOperations = new ArrayList<ExpressionType>();
    // optimizedOperations.add(ExpressionType.EQUALS);
    // optimizedOperations.add(ExpressionType.GREATERTHAN);
    // optimizedOperations.add(ExpressionType.GREATERTHAN_EQUALTO);
    // optimizedOperations.add(ExpressionType.LESSTHAN);
    // optimizedOperations.add(ExpressionType.LESSTHAN_EQUALTO);
    // optimizedOperations.add(ExpressionType.NOT);
    optimizedOperations.add(ExpressionType.TEXT_MATCH);
    this.dataMapMeta = new DataMapMeta(indexedColumns, optimizedOperations);
    // get analyzer
    // TODO: how to get analyzer ?
    analyzer = new StandardAnalyzer();
}

Example 4 with CarbonMetadata

use of org.apache.carbondata.core.metadata.CarbonMetadata in project carbondata by apache.

the class DictionaryClientTest method setUp.

@BeforeClass
public static void setUp() throws Exception {
    // enable lru cache by setting cache size
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "10");
    // Create two column schemas and dimensions for the table
    empColumnSchema = new ColumnSchema();
    empColumnSchema.setColumnName("empNameCol");
    empColumnSchema.setColumnUniqueId("empNameCol");
    empColumnSchema.setDimensionColumn(true);
    empColumnSchema.setDataType(DataTypes.STRING);
    empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
    empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
    ageColumnSchema = new ColumnSchema();
    ageColumnSchema.setColumnName("ageNameCol");
    ageColumnSchema.setDataType(DataTypes.SHORT_INT);
    ageColumnSchema.setColumnUniqueId("ageNameCol");
    ageColumnSchema.setDimensionColumn(true);
    ageColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
    ageDimension = new CarbonDimension(ageColumnSchema, 0, 0, 0, 0, 0);
    // Create a Table
    tableSchema = new TableSchema();
    tableSchema.setTableName("TestTable");
    tableSchema.setTableId("1");
    tableSchema.setListOfColumns(Arrays.asList(empColumnSchema, ageColumnSchema));
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    tableInfo = new TableInfo();
    tableInfo.setFactTable(tableSchema);
    tableInfo.setTableUniqueName("TestTable");
    tableInfo.setDatabaseName("test");
    storePath = System.getProperty("java.io.tmpdir") + "/tmp";
    tableInfo.setTablePath(storePath + "/" + "test" + "/" + "TestTable");
    CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
    // Add the created table to metadata
    metadata.addCarbonTable(carbonTable);
    // Start the server for testing the client
    server = NonSecureDictionaryServer.getInstance(5678, carbonTable);
    host = server.getHost();
}

Example 5 with CarbonMetadata

use of org.apache.carbondata.core.metadata.CarbonMetadata in project carbondata by apache.

the class IncrementalColumnDictionaryGeneratorTest method writeDictionaryData.

@Test
public void writeDictionaryData() throws Exception {
    // Create required column schema
    ColumnSchema columnSchema = new ColumnSchema();
    columnSchema.setColumnName("empNameCol");
    columnSchema.setDataType(DataTypes.STRING);
    columnSchema.setColumnUniqueId("empNameCol");
    CarbonDimension carbonDimension = new CarbonDimension(columnSchema, 0, 0, 0, 0, 0);
    // Create the generator and add the keys to dictionary
    IncrementalColumnDictionaryGenerator generator = new IncrementalColumnDictionaryGenerator(carbonDimension, 10, carbonTable);
    // Create a table schema for saving the dictionary
    TableSchema tableSchema = new TableSchema();
    tableSchema.setTableName("TestTable");
    tableSchema.setListOfColumns(Arrays.asList(columnSchema));
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    TableInfo tableInfo = new TableInfo();
    tableInfo.setFactTable(tableSchema);
    tableInfo.setTableUniqueName("TestTable");
    tableInfo.setDatabaseName("test");
    String storePath = System.getProperty("java.io.tmpdir") + "/tmp";
    File dictPath = new File(storePath + "/test/TestTable/Metadata/");
    System.out.print(dictPath.mkdirs());
    tableInfo.setTablePath(storePath + "/test" + "/TestTable");
    CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
    // Add the table to metadata
    metadata.addCarbonTable(carbonTable);
    // / Write the dictionary and verify whether its written successfully
    generator.writeDictionaryData();
    File dictionaryFile = new File(dictPath, "empNameCol.dict");
    System.out.println(dictionaryFile.getCanonicalPath());
    assertTrue(dictionaryFile.exists());
    dictionaryFile.delete();
    // cleanup created files
    metadata.removeTable(carbonTable.getTableUniqueName());
    cleanUpDirectory(new File(storePath));
}