Examples with SegmentStatus org.apache.carbondata.core.statusmanager.SegmentStatus used on opensource projects

Example 1 with SegmentStatus

use of org.apache.carbondata.core.statusmanager.SegmentStatus in project carbondata by apache.

the class CarbonUtil method calculateDataIndexSize.

/**
 * This method will calculate the data size and index size for carbon table
 */
public static Map<String, Long> calculateDataIndexSize(CarbonTable carbonTable) throws IOException {
    Map<String, Long> dataIndexSizeMap = new HashMap<String, Long>();
    long dataSize = 0L;
    long indexSize = 0L;
    long lastUpdateTime = 0L;
    boolean needUpdate = false;
    AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
    String isCalculated = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ENABLE_CALCULATE_SIZE, CarbonCommonConstants.DEFAULT_ENABLE_CALCULATE_SIZE);
    if (isCalculated.equalsIgnoreCase("true")) {
        SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
        ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
        try {
            if (carbonLock.lockWithRetries()) {
                LOGGER.info("Acquired lock for table for table status updation");
                String metadataPath = carbonTable.getMetadataPath();
                LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager.readLoadMetadata(metadataPath);
                for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
                    SegmentStatus loadStatus = loadMetadataDetail.getSegmentStatus();
                    if (loadStatus == SegmentStatus.SUCCESS || loadStatus == SegmentStatus.LOAD_PARTIAL_SUCCESS) {
                        String dsize = loadMetadataDetail.getDataSize();
                        String isize = loadMetadataDetail.getIndexSize();
                        // If it is old segment, need to calculate data size and index size again
                        if (null == dsize || null == isize) {
                            needUpdate = true;
                            LOGGER.info("It is an old segment, need calculate data size and index size again");
                            HashMap<String, Long> map = CarbonUtil.getDataSizeAndIndexSize(identifier.getTablePath(), loadMetadataDetail.getLoadName());
                            dsize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE));
                            isize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE));
                            loadMetadataDetail.setDataSize(dsize);
                            loadMetadataDetail.setIndexSize(isize);
                        }
                        dataSize += Long.parseLong(dsize);
                        indexSize += Long.parseLong(isize);
                    }
                }
                // If it contains old segment, write new load details
                if (needUpdate) {
                    SegmentStatusManager.writeLoadDetailsIntoFile(CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()), loadMetadataDetails);
                }
                String tableStatusPath = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
                if (FileFactory.isFileExist(tableStatusPath, FileFactory.getFileType(tableStatusPath))) {
                    lastUpdateTime = FileFactory.getCarbonFile(tableStatusPath, FileFactory.getFileType(tableStatusPath)).getLastModifiedTime();
                }
                dataIndexSizeMap.put(String.valueOf(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE), dataSize);
                dataIndexSizeMap.put(String.valueOf(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE), indexSize);
                dataIndexSizeMap.put(String.valueOf(CarbonCommonConstants.LAST_UPDATE_TIME), lastUpdateTime);
            } else {
                LOGGER.error("Not able to acquire the lock for Table status updation for table");
            }
        } finally {
            if (carbonLock.unlock()) {
                LOGGER.info("Table unlocked successfully after table status updation");
            } else {
                LOGGER.error("Unable to unlock Table lock for table during table status updation");
            }
        }
    }
    return dataIndexSizeMap;
}

Example 2 with SegmentStatus

use of org.apache.carbondata.core.statusmanager.SegmentStatus in project carbondata by apache.

the class CarbonUtil method calculateDataIndexSize.

/**
 * This method will calculate the data size and index size for carbon table
 */
public static Map<String, Long> calculateDataIndexSize(CarbonTable carbonTable, Boolean updateSize) throws IOException {
    Map<String, Long> dataIndexSizeMap = new HashMap<String, Long>();
    long totalDataSize = 0L;
    long totalIndexSize = 0L;
    long lastUpdateTime = 0L;
    boolean needUpdate = false;
    AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
    String isCalculated = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ENABLE_CALCULATE_SIZE, CarbonCommonConstants.DEFAULT_ENABLE_CALCULATE_SIZE);
    if (isCalculated.equalsIgnoreCase("true")) {
        SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
        ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
        try {
            boolean lockAcquired = true;
            if (updateSize) {
                lockAcquired = carbonLock.lockWithRetries();
            }
            if (lockAcquired) {
                LOGGER.debug("Acquired lock for table for table status update");
                String metadataPath = carbonTable.getMetadataPath();
                LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager.readLoadMetadata(metadataPath);
                for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
                    SegmentStatus loadStatus = loadMetadataDetail.getSegmentStatus();
                    if (loadStatus == SegmentStatus.SUCCESS || loadStatus == SegmentStatus.LOAD_PARTIAL_SUCCESS) {
                        String dataSize = loadMetadataDetail.getDataSize();
                        String indexSize = loadMetadataDetail.getIndexSize();
                        // If it is old segment, need to calculate data size and index size again
                        if (null == dataSize || null == indexSize) {
                            needUpdate = true;
                            LOGGER.debug("It is an old segment, need calculate data size and index size again");
                            HashMap<String, Long> map = CarbonUtil.getDataSizeAndIndexSize(identifier.getTablePath(), loadMetadataDetail.getLoadName());
                            dataSize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE));
                            indexSize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE));
                            loadMetadataDetail.setDataSize(dataSize);
                            loadMetadataDetail.setIndexSize(indexSize);
                        }
                        totalDataSize += Long.parseLong(dataSize);
                        totalIndexSize += Long.parseLong(indexSize);
                    }
                }
                // If it contains old segment, write new load details
                if (needUpdate && updateSize) {
                    SegmentStatusManager.writeLoadDetailsIntoFile(CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()), loadMetadataDetails);
                }
                String tableStatusPath = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
                if (FileFactory.isFileExist(tableStatusPath)) {
                    lastUpdateTime = FileFactory.getCarbonFile(tableStatusPath).getLastModifiedTime();
                }
                if (!FileFactory.isFileExist(metadataPath)) {
                    totalDataSize = FileFactory.getDirectorySize(carbonTable.getTablePath());
                }
                dataIndexSizeMap.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE, totalDataSize);
                dataIndexSizeMap.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE, totalIndexSize);
                dataIndexSizeMap.put(CarbonCommonConstants.LAST_UPDATE_TIME, lastUpdateTime);
            } else {
                LOGGER.error("Not able to acquire the lock for Table status update for table");
            }
        } finally {
            if (updateSize) {
                if (carbonLock.unlock()) {
                    LOGGER.debug("Table unlocked successfully after table status update");
                } else {
                    LOGGER.error("Unable to unlock Table lock for table during table status update");
                }
            }
        }
    }
    return dataIndexSizeMap;
}

Example 3 with SegmentStatus

use of org.apache.carbondata.core.statusmanager.SegmentStatus in project carbondata by apache.

the class CarbonLoaderUtil method updateTableStatusForFailure.

/**
 * This method will update the load failure entry in the table status file
 */
public static void updateTableStatusForFailure(CarbonLoadModel model, String uuid) throws IOException {
    // in case if failure the load status should be "Marked for delete" so that it will be taken
    // care during clean up
    SegmentStatus loadStatus = SegmentStatus.MARKED_FOR_DELETE;
    // always the last entry in the load metadata details will be the current load entry
    LoadMetadataDetails loadMetaEntry = model.getCurrentLoadMetadataDetail();
    if (loadMetaEntry == null) {
        return;
    }
    CarbonLoaderUtil.populateNewLoadMetaEntry(loadMetaEntry, loadStatus, model.getFactTimeStamp(), true);
    boolean entryAdded = CarbonLoaderUtil.recordNewLoadMetadata(loadMetaEntry, model, false, false, uuid, false);
    if (!entryAdded) {
        throw new IOException("Failed to update failure entry in table status for " + model.getTableName());
    }
}

Example 4 with SegmentStatus

use of org.apache.carbondata.core.statusmanager.SegmentStatus in project carbondata by apache.

the class CarbonLoaderUtil method readAndUpdateLoadProgressInTableMeta.

public static void readAndUpdateLoadProgressInTableMeta(CarbonLoadModel model, boolean insertOverwrite, String uuid) throws IOException {
    LoadMetadataDetails newLoadMetaEntry = new LoadMetadataDetails();
    SegmentStatus status = SegmentStatus.INSERT_IN_PROGRESS;
    if (insertOverwrite) {
        status = SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS;
    }
    // reading the start time of data load.
    if (model.getFactTimeStamp() == 0) {
        long loadStartTime = CarbonUpdateUtil.readCurrentTime();
        model.setFactTimeStamp(loadStartTime);
    }
    CarbonLoaderUtil.populateNewLoadMetaEntry(newLoadMetaEntry, status, model.getFactTimeStamp(), false);
    boolean entryAdded = CarbonLoaderUtil.recordNewLoadMetadata(newLoadMetaEntry, model, true, insertOverwrite, uuid, false);
    if (!entryAdded) {
        throw new IOException("Dataload failed due to failure in table status updation for " + model.getTableName());
    }
}