Examples with CarbonTableIdentifier org.apache.carbondata.core.metadata.CarbonTableIdentifier used on opensource projects

Example 6 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class IncrementalColumnDictionaryGenerator method writeDictionaryData.

@Override
public void writeDictionaryData(String tableUniqueName) throws IOException {
    // initialize params
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    CarbonTable carbonTable = metadata.getCarbonTable(tableUniqueName);
    CarbonTableIdentifier tableIdentifier = carbonTable.getCarbonTableIdentifier();
    ColumnIdentifier columnIdentifier = dimension.getColumnIdentifier();
    String storePath = carbonTable.getStorePath();
    DictionaryService dictionaryService = CarbonCommonFactory.getDictionaryService();
    // create dictionary cache from dictionary File
    DictionaryColumnUniqueIdentifier identifier = new DictionaryColumnUniqueIdentifier(tableIdentifier, columnIdentifier, columnIdentifier.getDataType());
    Boolean isDictExists = CarbonUtil.isFileExistsForGivenColumn(storePath, identifier);
    Dictionary dictionary = null;
    long t1 = System.currentTimeMillis();
    if (isDictExists) {
        Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, storePath);
        dictionary = dictCache.get(identifier);
    }
    long dictCacheTime = System.currentTimeMillis() - t1;
    long t2 = System.currentTimeMillis();
    // write dictionary
    CarbonDictionaryWriter dictionaryWriter = null;
    dictionaryWriter = dictionaryService.getDictionaryWriter(tableIdentifier, columnIdentifier, storePath);
    List<String> distinctValues = writeDictionary(dictionaryWriter, isDictExists);
    long dictWriteTime = System.currentTimeMillis() - t2;
    long t3 = System.currentTimeMillis();
    // write sort index
    if (distinctValues.size() > 0) {
        writeSortIndex(distinctValues, dictionary, dictionaryService, tableIdentifier, columnIdentifier, storePath);
    }
    long sortIndexWriteTime = System.currentTimeMillis() - t3;
    // update Meta Data
    updateMetaData(dictionaryWriter);
    LOGGER.audit("\n columnName: " + dimension.getColName() + "\n columnId: " + dimension.getColumnId() + "\n new distinct values count: " + distinctValues.size() + "\n create dictionary cache: " + dictCacheTime + "\n sort list, distinct and write: " + dictWriteTime + "\n write sort info: " + sortIndexWriteTime);
}

Example 7 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class FilterUtilTest method setUp.

@Before
public void setUp() throws Exception {
    init();
    this.databaseName = props.getProperty("database", "testSchema");
    this.tableName = props.getProperty("tableName", "carbon");
    this.carbonStorePath = props.getProperty("storePath", "carbonStore");
    carbonTableIdentifier = new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
    this.carbonStorePath = props.getProperty("storePath", "carbonStore");
    columnSchema = new ColumnSchema();
    columnSchema.setColumnar(true);
    columnSchema.setColumnName("IMEI");
    columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema.setDataType(DataType.STRING);
    columnSchema.setDimensionColumn(true);
}

Example 8 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class LocalFileLockTest method testingLocalFileLockingByAcquiring2Locks.

@Test
public void testingLocalFileLockingByAcquiring2Locks() {
    CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier("databaseName", "tableName", "tableId");
    LocalFileLock localLock1 = new LocalFileLock(carbonTableIdentifier, LockUsage.METADATA_LOCK);
    Assert.assertTrue(localLock1.lock());
    LocalFileLock localLock2 = new LocalFileLock(carbonTableIdentifier, LockUsage.METADATA_LOCK);
    Assert.assertTrue(!localLock2.lock());
    Assert.assertTrue(localLock1.unlock());
    Assert.assertTrue(localLock2.lock());
}

Example 9 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class CarbonDictionarySortIndexReaderImplTest method read.

/**
   * Test to read the data from dictionary sort index file
   *
   * @throws Exception
   */
@Test
public void read() throws Exception {
    deleteStorePath();
    CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier("testSchema", "carbon", UUID.randomUUID().toString());
    ColumnIdentifier columnIdentifier = new ColumnIdentifier("Name", null, null);
    CarbonDictionaryWriter dictionaryWriter = new CarbonDictionaryWriterImpl(storePath, carbonTableIdentifier, columnIdentifier);
    String metaFolderPath = storePath + File.separator + carbonTableIdentifier.getDatabaseName() + File.separator + carbonTableIdentifier.getTableName() + File.separator + "Metadata";
    CarbonUtil.checkAndCreateFolder(metaFolderPath);
    CarbonDictionarySortIndexWriter dictionarySortIndexWriter = new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, columnIdentifier, storePath);
    List<int[]> expectedData = prepareExpectedData();
    int[] data = expectedData.get(0);
    for (int i = 0; i < data.length; i++) {
        dictionaryWriter.write(String.valueOf(data[i]));
    }
    dictionaryWriter.close();
    dictionaryWriter.commit();
    List<Integer> sortIndex = Arrays.asList(ArrayUtils.toObject(expectedData.get(0)));
    List<Integer> invertedSortIndex = Arrays.asList(ArrayUtils.toObject(expectedData.get(1)));
    dictionarySortIndexWriter.writeSortIndex(sortIndex);
    dictionarySortIndexWriter.writeInvertedSortIndex(invertedSortIndex);
    dictionarySortIndexWriter.close();
    CarbonDictionarySortIndexReader dictionarySortIndexReader = new CarbonDictionarySortIndexReaderImpl(carbonTableIdentifier, columnIdentifier, storePath);
    List<Integer> actualSortIndex = dictionarySortIndexReader.readSortIndex();
    List<Integer> actualInvertedSortIndex = dictionarySortIndexReader.readInvertedSortIndex();
    for (int i = 0; i < actualSortIndex.size(); i++) {
        Assert.assertEquals(sortIndex.get(i), actualSortIndex.get(i));
        Assert.assertEquals(invertedSortIndex.get(i), actualInvertedSortIndex.get(i));
    }
}

Example 10 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class CarbonFactDataHandlerModel method getCarbonDataFolderLocation.

/**
   * This method will get the store location for the given path, segment id and partition id
   *
   * @return data directory path
   */
private static String getCarbonDataFolderLocation(CarbonDataLoadConfiguration configuration) {
    String carbonStorePath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS);
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableIdentifier.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + tableIdentifier.getTableName());
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTable.getCarbonTableIdentifier());
    return carbonTablePath.getCarbonDataDirectoryPath(configuration.getPartitionId(), configuration.getSegmentId() + "");
}