Examples with ICarbonLock org.apache.carbondata.core.locks.ICarbonLock used on opensource projects

Example 6 with ICarbonLock

use of org.apache.carbondata.core.locks.ICarbonLock in project carbondata by apache.

the class SegmentStatusManager method updateDeletionStatus.

/**
   * updates deletion status
   *
   * @param loadIds
   * @param tableFolderPath
   * @return
   */
public static List<String> updateDeletionStatus(AbsoluteTableIdentifier identifier, List<String> loadIds, String tableFolderPath) throws Exception {
    CarbonTableIdentifier carbonTableIdentifier = identifier.getCarbonTableIdentifier();
    ICarbonLock carbonDeleteSegmentLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier, LockUsage.DELETE_SEGMENT_LOCK);
    ICarbonLock carbonTableStatusLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier, LockUsage.TABLE_STATUS_LOCK);
    String tableDetails = carbonTableIdentifier.getDatabaseName() + "." + carbonTableIdentifier.getTableName();
    List<String> invalidLoadIds = new ArrayList<String>(0);
    try {
        if (carbonDeleteSegmentLock.lockWithRetries()) {
            LOG.info("Delete segment lock has been successfully acquired");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier.getStorePath(), identifier.getCarbonTableIdentifier());
            String dataLoadLocation = carbonTablePath.getTableStatusFilePath();
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = null;
            if (!FileFactory.isFileExist(dataLoadLocation, FileFactory.getFileType(dataLoadLocation))) {
                // log error.
                LOG.error("Load metadata file is not present.");
                return loadIds;
            }
            // read existing metadata details in load metadata.
            listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath);
            if (listOfLoadFolderDetailsArray != null && listOfLoadFolderDetailsArray.length != 0) {
                updateDeletionStatus(loadIds, listOfLoadFolderDetailsArray, invalidLoadIds);
                if (invalidLoadIds.isEmpty()) {
                    // All or None , if anything fails then dont write
                    if (carbonTableStatusLock.lockWithRetries()) {
                        LOG.info("Table status lock has been successfully acquired");
                        // To handle concurrency scenarios, always take latest metadata before writing
                        // into status file.
                        LoadMetadataDetails[] latestLoadMetadataDetails = readLoadMetadata(tableFolderPath);
                        updateLatestTableStatusDetails(listOfLoadFolderDetailsArray, latestLoadMetadataDetails);
                        writeLoadDetailsIntoFile(dataLoadLocation, listOfLoadFolderDetailsArray);
                    } else {
                        String errorMsg = "Delete segment by id is failed for " + tableDetails + ". Not able to acquire the table status lock due to other operation running " + "in the background.";
                        LOG.audit(errorMsg);
                        LOG.error(errorMsg);
                        throw new Exception(errorMsg + " Please try after some time.");
                    }
                } else {
                    return invalidLoadIds;
                }
            } else {
                LOG.audit("Delete segment by Id is failed. No matching segment id found.");
                return loadIds;
            }
        } else {
            String errorMsg = "Delete segment by id is failed for " + tableDetails + ". Not able to acquire the delete segment lock due to another delete " + "operation is running in the background.";
            LOG.audit(errorMsg);
            LOG.error(errorMsg);
            throw new Exception(errorMsg + " Please try after some time.");
        }
    } catch (IOException e) {
        LOG.error("IOException" + e.getMessage());
        throw e;
    } finally {
        CarbonLockUtil.fileUnlock(carbonTableStatusLock, LockUsage.TABLE_STATUS_LOCK);
        CarbonLockUtil.fileUnlock(carbonDeleteSegmentLock, LockUsage.DELETE_SEGMENT_LOCK);
    }
    return invalidLoadIds;
}

Example 7 with ICarbonLock

use of org.apache.carbondata.core.locks.ICarbonLock in project carbondata by apache.

the class CarbonDataMergerUtil method updateStatusFile.

public static Boolean updateStatusFile(List<CarbonDataMergerUtilResult> updateDataMergerDetailsList, CarbonTable table, String timestamp, SegmentUpdateStatusManager segmentUpdateStatusManager) {
    List<SegmentUpdateDetails> segmentUpdateDetails = new ArrayList<SegmentUpdateDetails>(updateDataMergerDetailsList.size());
    // Check the list output.
    for (CarbonDataMergerUtilResult carbonDataMergerUtilResult : updateDataMergerDetailsList) {
        if (carbonDataMergerUtilResult.getCompactionStatus()) {
            SegmentUpdateDetails tempSegmentUpdateDetails = new SegmentUpdateDetails();
            tempSegmentUpdateDetails.setSegmentName(carbonDataMergerUtilResult.getSegmentName());
            tempSegmentUpdateDetails.setBlockName(carbonDataMergerUtilResult.getBlockName());
            for (SegmentUpdateDetails origDetails : segmentUpdateStatusManager.getUpdateStatusDetails()) {
                if (origDetails.getBlockName().equalsIgnoreCase(carbonDataMergerUtilResult.getBlockName()) && origDetails.getSegmentName().equalsIgnoreCase(carbonDataMergerUtilResult.getSegmentName())) {
                    tempSegmentUpdateDetails.setDeletedRowsInBlock(origDetails.getDeletedRowsInBlock());
                    tempSegmentUpdateDetails.setStatus(origDetails.getStatus());
                    break;
                }
            }
            tempSegmentUpdateDetails.setDeleteDeltaStartTimestamp(carbonDataMergerUtilResult.getDeleteDeltaStartTimestamp());
            tempSegmentUpdateDetails.setDeleteDeltaEndTimestamp(carbonDataMergerUtilResult.getDeleteDeltaEndTimestamp());
            segmentUpdateDetails.add(tempSegmentUpdateDetails);
        } else
            return false;
    }
    CarbonUpdateUtil.updateSegmentStatus(segmentUpdateDetails, table, timestamp, true);
    // Update the Table Status.
    String metaDataFilepath = table.getMetaDataFilepath();
    AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String tableStatusPath = carbonTablePath.getTableStatusFilePath();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    boolean lockStatus = false;
    try {
        lockStatus = carbonLock.lockWithRetries();
        if (lockStatus) {
            LOGGER.info("Acquired lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " for table status updation");
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = segmentStatusManager.readLoadMetadata(metaDataFilepath);
            for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) {
                if (loadMetadata.getLoadName().equalsIgnoreCase("0")) {
                    loadMetadata.setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
                }
            }
            try {
                segmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetailsArray);
            } catch (IOException e) {
                return false;
            }
        } else {
            LOGGER.error("Not able to acquire the lock for Table status updation for table " + table.getDatabaseName() + "." + table.getFactTableName());
        }
    } finally {
        if (lockStatus) {
            if (carbonLock.unlock()) {
                LOGGER.info("Table unlocked successfully after table status updation" + table.getDatabaseName() + "." + table.getFactTableName());
            } else {
                LOGGER.error("Unable to unlock Table lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " during table status updation");
            }
        }
    }
    return true;
}

Example 8 with ICarbonLock

use of org.apache.carbondata.core.locks.ICarbonLock in project carbondata by apache.

the class CarbonUpdateUtil method updateSegmentStatus.

/**
   * @param updateDetailsList
   * @param table
   * @param updateStatusFileIdentifier
   * @return
   */
public static boolean updateSegmentStatus(List<SegmentUpdateDetails> updateDetailsList, CarbonTable table, String updateStatusFileIdentifier, boolean isCompaction) {
    boolean status = false;
    SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(table.getAbsoluteTableIdentifier());
    ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock();
    boolean lockStatus = false;
    try {
        lockStatus = updateLock.lockWithRetries();
        if (lockStatus) {
            AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
            // read the existing file if present and update the same.
            SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager.getUpdateStatusDetails();
            List<SegmentUpdateDetails> oldList = new ArrayList(Arrays.asList(oldDetails));
            for (SegmentUpdateDetails newBlockEntry : updateDetailsList) {
                int index = oldList.indexOf(newBlockEntry);
                if (index != -1) {
                    // update the element in existing list.
                    SegmentUpdateDetails blockDetail = oldList.get(index);
                    if (blockDetail.getDeleteDeltaStartTimestamp().isEmpty() || (isCompaction == true)) {
                        blockDetail.setDeleteDeltaStartTimestamp(newBlockEntry.getDeleteDeltaStartTimestamp());
                    }
                    blockDetail.setDeleteDeltaEndTimestamp(newBlockEntry.getDeleteDeltaEndTimestamp());
                    blockDetail.setStatus(newBlockEntry.getStatus());
                    blockDetail.setDeletedRowsInBlock(newBlockEntry.getDeletedRowsInBlock());
                } else {
                    // add the new details to the list.
                    oldList.add(newBlockEntry);
                }
            }
            segmentUpdateStatusManager.writeLoadDetailsIntoFile(oldList, updateStatusFileIdentifier);
            status = true;
        } else {
            LOGGER.error("Not able to acquire the segment update lock.");
            status = false;
        }
    } catch (IOException e) {
        status = false;
    } finally {
        if (lockStatus) {
            if (updateLock.unlock()) {
                LOGGER.info("Unlock the segment update lock successfull.");
            } else {
                LOGGER.error("Not able to unlock the segment update lock.");
            }
        }
    }
    return status;
}

Example 9 with ICarbonLock

use of org.apache.carbondata.core.locks.ICarbonLock in project carbondata by apache.

the class CarbonUpdateUtil method updateTableMetadataStatus.

/**
   *
   * @param updatedSegmentsList
   * @param table
   * @param updatedTimeStamp
   * @param isTimestampUpdationRequired
   * @param segmentsToBeDeleted
   * @return
   */
public static boolean updateTableMetadataStatus(Set<String> updatedSegmentsList, CarbonTable table, String updatedTimeStamp, boolean isTimestampUpdationRequired, List<String> segmentsToBeDeleted) {
    boolean status = false;
    String metaDataFilepath = table.getMetaDataFilepath();
    AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String tableStatusPath = carbonTablePath.getTableStatusFilePath();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    boolean lockStatus = false;
    try {
        lockStatus = carbonLock.lockWithRetries();
        if (lockStatus) {
            LOGGER.info("Acquired lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " for table status updation");
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = segmentStatusManager.readLoadMetadata(metaDataFilepath);
            for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) {
                if (isTimestampUpdationRequired) {
                    // we are storing the link between the 2 status files in the segment 0 only.
                    if (loadMetadata.getLoadName().equalsIgnoreCase("0")) {
                        loadMetadata.setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(updatedTimeStamp));
                    }
                    // if the segments is in the list of marked for delete then update the status.
                    if (segmentsToBeDeleted.contains(loadMetadata.getLoadName())) {
                        loadMetadata.setLoadStatus(CarbonCommonConstants.MARKED_FOR_DELETE);
                        loadMetadata.setModificationOrdeletionTimesStamp(Long.parseLong(updatedTimeStamp));
                    }
                }
                for (String segName : updatedSegmentsList) {
                    if (loadMetadata.getLoadName().equalsIgnoreCase(segName)) {
                        // String will come empty then no need to write into table status file.
                        if (isTimestampUpdationRequired) {
                            loadMetadata.setIsDeleted(CarbonCommonConstants.KEYWORD_TRUE);
                            // if in case of update flow.
                            if (loadMetadata.getUpdateDeltaStartTimestamp().isEmpty()) {
                                // this means for first time it is getting updated .
                                loadMetadata.setUpdateDeltaStartTimestamp(updatedTimeStamp);
                            }
                            // update end timestamp for each time.
                            loadMetadata.setUpdateDeltaEndTimestamp(updatedTimeStamp);
                        }
                    }
                }
            }
            try {
                segmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetailsArray);
            } catch (IOException e) {
                return false;
            }
            status = true;
        } else {
            LOGGER.error("Not able to acquire the lock for Table status updation for table " + table.getDatabaseName() + "." + table.getFactTableName());
        }
    } finally {
        if (lockStatus) {
            if (carbonLock.unlock()) {
                LOGGER.info("Table unlocked successfully after table status updation" + table.getDatabaseName() + "." + table.getFactTableName());
            } else {
                LOGGER.error("Unable to unlock Table lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " during table status updation");
            }
        }
    }
    return status;
}