Examples with CarbonTableIdentifier org.apache.carbondata.core.metadata.CarbonTableIdentifier used on opensource projects

Example 1 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class IncrementalColumnDictionaryGenerator method writeDictionaryData.

@Override
public void writeDictionaryData(String tableUniqueName) throws IOException {
    // initialize params
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    CarbonTable carbonTable = metadata.getCarbonTable(tableUniqueName);
    CarbonTableIdentifier tableIdentifier = carbonTable.getCarbonTableIdentifier();
    ColumnIdentifier columnIdentifier = dimension.getColumnIdentifier();
    String storePath = carbonTable.getStorePath();
    DictionaryService dictionaryService = CarbonCommonFactory.getDictionaryService();
    // create dictionary cache from dictionary File
    DictionaryColumnUniqueIdentifier identifier = new DictionaryColumnUniqueIdentifier(tableIdentifier, columnIdentifier, columnIdentifier.getDataType());
    Boolean isDictExists = CarbonUtil.isFileExistsForGivenColumn(storePath, identifier);
    Dictionary dictionary = null;
    long t1 = System.currentTimeMillis();
    if (isDictExists) {
        Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, storePath);
        dictionary = dictCache.get(identifier);
    }
    long dictCacheTime = System.currentTimeMillis() - t1;
    long t2 = System.currentTimeMillis();
    // write dictionary
    CarbonDictionaryWriter dictionaryWriter = null;
    dictionaryWriter = dictionaryService.getDictionaryWriter(tableIdentifier, columnIdentifier, storePath);
    List<String> distinctValues = writeDictionary(dictionaryWriter, isDictExists);
    long dictWriteTime = System.currentTimeMillis() - t2;
    long t3 = System.currentTimeMillis();
    // write sort index
    if (distinctValues.size() > 0) {
        writeSortIndex(distinctValues, dictionary, dictionaryService, tableIdentifier, columnIdentifier, storePath);
    }
    long sortIndexWriteTime = System.currentTimeMillis() - t3;
    // update Meta Data
    updateMetaData(dictionaryWriter);
    LOGGER.audit("\n columnName: " + dimension.getColName() + "\n columnId: " + dimension.getColumnId() + "\n new distinct values count: " + distinctValues.size() + "\n create dictionary cache: " + dictCacheTime + "\n sort list, distinct and write: " + dictWriteTime + "\n write sort info: " + sortIndexWriteTime);
}

Example 2 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class CarbonDataProcessorUtil method checkAndCreateCarbonStoreLocation.

/**
   * This method will get the store location for the given path, segment id and partition id
   *
   * @return data directory path
   */
public static String checkAndCreateCarbonStoreLocation(String factStoreLocation, String databaseName, String tableName, String partitionId, String segmentId) {
    CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName + CarbonCommonConstants.UNDERSCORE + tableName);
    CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(factStoreLocation, carbonTableIdentifier);
    String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId);
    CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
    return carbonDataDirectoryPath;
}

Example 3 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class BlockIndexStoreTest method testLoadAndGetTaskIdToSegmentsMapForSingleSegment.

@Test
public void testLoadAndGetTaskIdToSegmentsMapForSingleSegment() throws IOException {
    File file = getPartFile();
    TableBlockInfo info = new TableBlockInfo(file.getAbsolutePath(), 0, "0", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "t3", "1");
    AbsoluteTableIdentifier absoluteTableIdentifier = new AbsoluteTableIdentifier("/src/test/resources", carbonTableIdentifier);
    try {
        List<TableBlockUniqueIdentifier> tableBlockInfoList = getTableBlockUniqueIdentifierList(Arrays.asList(new TableBlockInfo[] { info }), absoluteTableIdentifier);
        List<AbstractIndex> loadAndGetBlocks = cache.getAll(tableBlockInfoList);
        assertTrue(loadAndGetBlocks.size() == 1);
    } catch (Exception e) {
        assertTrue(false);
    }
    List<String> segmentIds = new ArrayList<>();
    segmentIds.add(info.getSegmentId());
    cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
}

Example 4 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class DeleteLoadFolders method getSegmentPath.

/**
   * returns segment path
   *
   * @param dbName
   * @param tableName
   * @param storeLocation
   * @param partitionId
   * @param oneLoad
   * @return
   */
private static String getSegmentPath(String dbName, String tableName, String storeLocation, int partitionId, LoadMetadataDetails oneLoad) {
    CarbonTablePath carbon = new CarbonStorePath(storeLocation).getCarbonTablePath(new CarbonTableIdentifier(dbName, tableName, ""));
    String segmentId = oneLoad.getLoadName();
    return carbon.getCarbonDataDirectoryPath("" + partitionId, segmentId);
}

Example 5 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class DataLoadExecutor method execute.

public void execute(CarbonLoadModel loadModel, String storeLocation, CarbonIterator<Object[]>[] inputIterators) throws Exception {
    AbstractDataLoadProcessorStep loadProcessorStep = null;
    try {
        loadProcessorStep = new DataLoadProcessBuilder().build(loadModel, storeLocation, inputIterators);
        // 1. initialize
        loadProcessorStep.initialize();
        LOGGER.info("Data Loading is started for table " + loadModel.getTableName());
        // 2. execute the step
        loadProcessorStep.execute();
    } catch (CarbonDataLoadingException e) {
        throw e;
    } catch (Exception e) {
        LOGGER.error(e, "Data Loading failed for table " + loadModel.getTableName());
        throw new CarbonDataLoadingException("Data Loading failed for table " + loadModel.getTableName(), e);
    } finally {
        if (loadProcessorStep != null) {
            // 3. Close the step
            loadProcessorStep.close();
        }
    }
    String key = new CarbonTableIdentifier(loadModel.getDatabaseName(), loadModel.getTableName(), null).getBadRecordLoggerKey();
    if (null != BadRecordsLogger.hasBadRecord(key)) {
        LOGGER.error("Data Load is partially success for table " + loadModel.getTableName());
        throw new BadRecordFoundException("Bad records found during load");
    } else {
        LOGGER.info("Data loading is successful for table " + loadModel.getTableName());
    }
}