Examples with CarbonTableIdentifier org.apache.carbondata.core.metadata.CarbonTableIdentifier used on opensource projects

Example 36 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class DataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    Iterator<CarbonRowBatch>[] iterators = child.execute();
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    String tableName = tableIdentifier.getTableName();
    try {
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(), System.currentTimeMillis());
        int i = 0;
        for (Iterator<CarbonRowBatch> iterator : iterators) {
            String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(i));
            CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, i, 0);
            CarbonFactHandler dataHandler = null;
            boolean rowsNotExist = true;
            while (iterator.hasNext()) {
                if (rowsNotExist) {
                    rowsNotExist = false;
                    dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
                    dataHandler.initialise();
                }
                processBatch(iterator.next(), dataHandler);
            }
            if (!rowsNotExist) {
                finish(tableName, dataHandler);
            }
            i++;
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("Error while initializing data handler : " + e.getMessage());
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
    }
    return null;
}

Example 37 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class SegmentStatusManager method updateDeletionStatus.

/**
   * updates deletion status
   *
   * @param loadIds
   * @param tableFolderPath
   * @return
   */
public static List<String> updateDeletionStatus(AbsoluteTableIdentifier identifier, List<String> loadIds, String tableFolderPath) throws Exception {
    CarbonTableIdentifier carbonTableIdentifier = identifier.getCarbonTableIdentifier();
    ICarbonLock carbonDeleteSegmentLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier, LockUsage.DELETE_SEGMENT_LOCK);
    ICarbonLock carbonTableStatusLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier, LockUsage.TABLE_STATUS_LOCK);
    String tableDetails = carbonTableIdentifier.getDatabaseName() + "." + carbonTableIdentifier.getTableName();
    List<String> invalidLoadIds = new ArrayList<String>(0);
    try {
        if (carbonDeleteSegmentLock.lockWithRetries()) {
            LOG.info("Delete segment lock has been successfully acquired");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier.getStorePath(), identifier.getCarbonTableIdentifier());
            String dataLoadLocation = carbonTablePath.getTableStatusFilePath();
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = null;
            if (!FileFactory.isFileExist(dataLoadLocation, FileFactory.getFileType(dataLoadLocation))) {
                // log error.
                LOG.error("Load metadata file is not present.");
                return loadIds;
            }
            // read existing metadata details in load metadata.
            listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath);
            if (listOfLoadFolderDetailsArray != null && listOfLoadFolderDetailsArray.length != 0) {
                updateDeletionStatus(loadIds, listOfLoadFolderDetailsArray, invalidLoadIds);
                if (invalidLoadIds.isEmpty()) {
                    // All or None , if anything fails then dont write
                    if (carbonTableStatusLock.lockWithRetries()) {
                        LOG.info("Table status lock has been successfully acquired");
                        // To handle concurrency scenarios, always take latest metadata before writing
                        // into status file.
                        LoadMetadataDetails[] latestLoadMetadataDetails = readLoadMetadata(tableFolderPath);
                        updateLatestTableStatusDetails(listOfLoadFolderDetailsArray, latestLoadMetadataDetails);
                        writeLoadDetailsIntoFile(dataLoadLocation, listOfLoadFolderDetailsArray);
                    } else {
                        String errorMsg = "Delete segment by id is failed for " + tableDetails + ". Not able to acquire the table status lock due to other operation running " + "in the background.";
                        LOG.audit(errorMsg);
                        LOG.error(errorMsg);
                        throw new Exception(errorMsg + " Please try after some time.");
                    }
                } else {
                    return invalidLoadIds;
                }
            } else {
                LOG.audit("Delete segment by Id is failed. No matching segment id found.");
                return loadIds;
            }
        } else {
            String errorMsg = "Delete segment by id is failed for " + tableDetails + ". Not able to acquire the delete segment lock due to another delete " + "operation is running in the background.";
            LOG.audit(errorMsg);
            LOG.error(errorMsg);
            throw new Exception(errorMsg + " Please try after some time.");
        }
    } catch (IOException e) {
        LOG.error("IOException" + e.getMessage());
        throw e;
    } finally {
        CarbonLockUtil.fileUnlock(carbonTableStatusLock, LockUsage.TABLE_STATUS_LOCK);
        CarbonLockUtil.fileUnlock(carbonDeleteSegmentLock, LockUsage.DELETE_SEGMENT_LOCK);
    }
    return invalidLoadIds;
}

Example 38 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class CarbonDictionaryReaderImplTest method setUp.

@BeforeClass
public static void setUp() throws Exception {
    columnIdentifier = new ColumnIdentifier("1", null, null);
    carbonTableIdentifier = new CarbonTableIdentifier("dbName", "tableName", UUID.randomUUID().toString());
    carbonDictionaryReaderImpl = new CarbonDictionaryReaderImpl("storePath", carbonTableIdentifier, columnIdentifier);
}

Example 39 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class CarbonTableReader method parseCarbonMetadata.

/**
   * parse carbon metadata into cc(CarbonTableReader cache)
   */
public CarbonTable parseCarbonMetadata(SchemaTableName table) {
    CarbonTable result = null;
    try {
        CarbonTableCacheModel cache = cc.getOrDefault(table, new CarbonTableCacheModel());
        if (cache.isValid())
            return cache.carbonTable;
        //Step1: get table meta path, load carbon table param
        String storePath = config.getStorePath();
        cache.carbonTableIdentifier = new CarbonTableIdentifier(table.getSchemaName(), table.getTableName(), UUID.randomUUID().toString());
        cache.carbonTablePath = PathFactory.getInstance().getCarbonTablePath(storePath, cache.carbonTableIdentifier);
        cc.put(table, cache);
        //Step2: check file existed? read schema file
        ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {

            public TBase create() {
                return new org.apache.carbondata.format.TableInfo();
            }
        };
        ThriftReader thriftReader = new ThriftReader(cache.carbonTablePath.getSchemaFilePath(), createTBase);
        thriftReader.open();
        org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
        thriftReader.close();
        // Step3: Transform Format Level TableInfo to Code Level TableInfo
        SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
        TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo, table.getSchemaName(), table.getTableName(), storePath);
        wrapperTableInfo.setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(cache.carbonTablePath.getSchemaFilePath()));
        // Step4: Load metadata info into CarbonMetadata
        CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
        cache.tableInfo = wrapperTableInfo;
        cache.carbonTable = CarbonMetadata.getInstance().getCarbonTable(cache.carbonTableIdentifier.getTableUniqueName());
        result = cache.carbonTable;
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
    return result;
}

Example 40 with CarbonTableIdentifier

use of org.apache.carbondata.core.metadata.CarbonTableIdentifier in project carbondata by apache.

the class CarbonLoaderUtil method checkAndCreateCarbonDataLocation.

/**
   * This method will get the store location for the given path, segment id and partition id
   *
   * @param carbonStorePath
   * @param segmentId
   */
public static void checkAndCreateCarbonDataLocation(String carbonStorePath, String segmentId, CarbonTable carbonTable) {
    CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTableIdentifier);
    String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
}