Examples with ICarbonLock org.apache.carbondata.core.locks.ICarbonLock used on opensource projects

Example 1 with ICarbonLock

use of org.apache.carbondata.core.locks.ICarbonLock in project carbondata by apache.

the class SegmentStatusManager method updateDeletionStatus.

/**
   * updates deletion status
   *
   * @param loadDate
   * @param tableFolderPath
   * @return
   */
public static List<String> updateDeletionStatus(AbsoluteTableIdentifier identifier, String loadDate, String tableFolderPath, Long loadStartTime) throws Exception {
    CarbonTableIdentifier carbonTableIdentifier = identifier.getCarbonTableIdentifier();
    ICarbonLock carbonDeleteSegmentLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier, LockUsage.DELETE_SEGMENT_LOCK);
    ICarbonLock carbonTableStatusLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier, LockUsage.TABLE_STATUS_LOCK);
    String tableDetails = carbonTableIdentifier.getDatabaseName() + "." + carbonTableIdentifier.getTableName();
    List<String> invalidLoadTimestamps = new ArrayList<String>(0);
    try {
        if (carbonDeleteSegmentLock.lockWithRetries()) {
            LOG.info("Delete segment lock has been successfully acquired");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier.getStorePath(), identifier.getCarbonTableIdentifier());
            String dataLoadLocation = carbonTablePath.getTableStatusFilePath();
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = null;
            if (!FileFactory.isFileExist(dataLoadLocation, FileFactory.getFileType(dataLoadLocation))) {
                // log error.
                LOG.error("Error message: " + "Load metadata file is not present.");
                invalidLoadTimestamps.add(loadDate);
                return invalidLoadTimestamps;
            }
            // read existing metadata details in load metadata.
            listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath);
            if (listOfLoadFolderDetailsArray != null && listOfLoadFolderDetailsArray.length != 0) {
                updateDeletionStatus(loadDate, listOfLoadFolderDetailsArray, invalidLoadTimestamps, loadStartTime);
                if (invalidLoadTimestamps.isEmpty()) {
                    if (carbonTableStatusLock.lockWithRetries()) {
                        LOG.info("Table status lock has been successfully acquired.");
                        // To handle concurrency scenarios, always take latest metadata before writing
                        // into status file.
                        LoadMetadataDetails[] latestLoadMetadataDetails = readLoadMetadata(tableFolderPath);
                        updateLatestTableStatusDetails(listOfLoadFolderDetailsArray, latestLoadMetadataDetails);
                        writeLoadDetailsIntoFile(dataLoadLocation, listOfLoadFolderDetailsArray);
                    } else {
                        String errorMsg = "Delete segment by date is failed for " + tableDetails + ". Not able to acquire the table status lock due to other operation running " + "in the background.";
                        LOG.audit(errorMsg);
                        LOG.error(errorMsg);
                        throw new Exception(errorMsg + " Please try after some time.");
                    }
                } else {
                    return invalidLoadTimestamps;
                }
            } else {
                LOG.audit("Delete segment by date is failed. No matching segment found.");
                invalidLoadTimestamps.add(loadDate);
                return invalidLoadTimestamps;
            }
        } else {
            String errorMsg = "Delete segment by date is failed for " + tableDetails + ". Not able to acquire the delete segment lock due to another delete " + "operation is running in the background.";
            LOG.audit(errorMsg);
            LOG.error(errorMsg);
            throw new Exception(errorMsg + " Please try after some time.");
        }
    } catch (IOException e) {
        LOG.error("Error message: " + "IOException" + e.getMessage());
        throw e;
    } finally {
        CarbonLockUtil.fileUnlock(carbonTableStatusLock, LockUsage.TABLE_STATUS_LOCK);
        CarbonLockUtil.fileUnlock(carbonDeleteSegmentLock, LockUsage.DELETE_SEGMENT_LOCK);
    }
    return invalidLoadTimestamps;
}

Example 2 with ICarbonLock

use of org.apache.carbondata.core.locks.ICarbonLock in project carbondata by apache.

the class CarbonDataMergerUtil method updateMajorCompactionPropertyInSegment.

/**
   * This will update the property of segments as major compacted.
   * @param model
   * @param changedSegDetails
   */
public static void updateMajorCompactionPropertyInSegment(CarbonLoadModel model, List<LoadMetadataDetails> changedSegDetails, List<LoadMetadataDetails> preservedSegment) throws Exception {
    String metadataPath = model.getCarbonDataLoadSchema().getCarbonTable().getMetaDataFilepath();
    AbsoluteTableIdentifier absoluteTableIdentifier = model.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    LoadMetadataDetails[] details = segmentStatusManager.readLoadMetadata(metadataPath);
    List<LoadMetadataDetails> originalList = Arrays.asList(details);
    for (LoadMetadataDetails segment : changedSegDetails) {
        if (preservedSegment.contains(segment)) {
            continue;
        }
        originalList.get(originalList.indexOf(segment)).setMajorCompacted("true");
    }
    ICarbonLock carbonTableStatusLock = CarbonLockFactory.getCarbonLockObj(model.getCarbonDataLoadSchema().getCarbonTable().getCarbonTableIdentifier(), LockUsage.TABLE_STATUS_LOCK);
    try {
        if (carbonTableStatusLock.lockWithRetries()) {
            LOGGER.info("Acquired lock for the table " + model.getDatabaseName() + "." + model.getTableName() + " for table status updation ");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
            segmentStatusManager.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(), originalList.toArray(new LoadMetadataDetails[originalList.size()]));
        } else {
            LOGGER.error("Could not able to obtain lock for table" + model.getDatabaseName() + "." + model.getTableName() + "for table status updation");
            throw new Exception("Failed to update the MajorCompactionStatus.");
        }
    } catch (IOException e) {
        LOGGER.error("Error while writing metadata");
        throw new Exception("Failed to update the MajorCompactionStatus." + e.getMessage());
    } finally {
        if (carbonTableStatusLock.unlock()) {
            LOGGER.info("Table unlocked successfully after table status updation" + model.getDatabaseName() + "." + model.getTableName());
        } else {
            LOGGER.error("Unable to unlock Table lock for table" + model.getDatabaseName() + "." + model.getTableName() + " during table status updation");
        }
    }
}

Example 3 with ICarbonLock

use of org.apache.carbondata.core.locks.ICarbonLock in project carbondata by apache.

the class CarbonDataMergerUtil method updateLoadMetadataIUDUpdateDeltaMergeStatus.

/**
   * Update Both Segment Update Status and Table Status for the case of IUD Delete
   * delta compaction.
   *
   * @param loadsToMerge
   * @param metaDataFilepath
   * @param carbonLoadModel
   * @return
   */
public static boolean updateLoadMetadataIUDUpdateDeltaMergeStatus(List<LoadMetadataDetails> loadsToMerge, String metaDataFilepath, CarbonLoadModel carbonLoadModel) {
    boolean status = false;
    boolean updateLockStatus = false;
    boolean tableLockStatus = false;
    String timestamp = "" + carbonLoadModel.getFactTimeStamp();
    List<String> updatedDeltaFilesList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    // This routine updateLoadMetadataIUDCompactionMergeStatus is suppose to update
    // two files as it is only called during IUD_UPDDEL_DELTA_COMPACTION. Along with
    // Table Status Metadata file (For Update Block Compaction) it has to update the
    // Table Update Status Metadata File (For corresponding Delete Delta File).
    // As the IUD_UPDDEL_DELTA_COMPACTION going to write in the same segment therefore in
    // A) Table Update Status Metadata File (Block Level)
    //      * For each blocks which is being compacted Mark 'Compacted' as the Status.
    // B) Table Status Metadata file (Segment Level)
    //      * loadStatus won't be changed to "compacted'
    //      * UpdateDeltaStartTime and UpdateDeltaEndTime will be both set to current
    //        timestamp (which is being passed from driver)
    // First the Table Update Status Metadata File should be updated as we need to get
    // the updated blocks for the segment from Table Status Metadata Update Delta Start and
    // End Timestamp.
    // Table Update Status Metadata Update.
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(absoluteTableIdentifier);
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock();
    ICarbonLock statusLock = segmentStatusManager.getTableStatusLock();
    // Update the Compacted Blocks with Compacted Status.
    try {
        updatedDeltaFilesList = segmentUpdateStatusManager.getUpdateDeltaFiles(loadsToMerge.get(0).getLoadName().toString());
    } catch (Exception e) {
        LOGGER.error("Error while getting the Update Delta Blocks.");
        status = false;
        return status;
    }
    if (updatedDeltaFilesList.size() > 0) {
        try {
            updateLockStatus = updateLock.lockWithRetries();
            tableLockStatus = statusLock.lockWithRetries();
            List<String> blockNames = new ArrayList<>(updatedDeltaFilesList.size());
            for (String compactedBlocks : updatedDeltaFilesList) {
                // Try to BlockName
                int endIndex = compactedBlocks.lastIndexOf(File.separator);
                String blkNoExt = compactedBlocks.substring(endIndex + 1, compactedBlocks.lastIndexOf("-"));
                blockNames.add(blkNoExt);
            }
            if (updateLockStatus && tableLockStatus) {
                SegmentUpdateDetails[] updateLists = segmentUpdateStatusManager.readLoadMetadata();
                for (String compactedBlocks : blockNames) {
                    // Check is the compactedBlocks name matches with oldDetails
                    for (int i = 0; i < updateLists.length; i++) {
                        if (updateLists[i].getBlockName().equalsIgnoreCase(compactedBlocks) && !CarbonCommonConstants.COMPACTED.equalsIgnoreCase(updateLists[i].getStatus()) && !CarbonCommonConstants.MARKED_FOR_DELETE.equalsIgnoreCase(updateLists[i].getStatus())) {
                            updateLists[i].setStatus(CarbonCommonConstants.COMPACTED);
                        }
                    }
                }
                LoadMetadataDetails[] loadDetails = segmentStatusManager.readLoadMetadata(metaDataFilepath);
                for (LoadMetadataDetails loadDetail : loadDetails) {
                    if (loadsToMerge.contains(loadDetail)) {
                        loadDetail.setUpdateDeltaStartTimestamp(timestamp);
                        loadDetail.setUpdateDeltaEndTimestamp(timestamp);
                        if (loadDetail.getLoadName().equalsIgnoreCase("0")) {
                            loadDetail.setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
                        }
                    }
                }
                try {
                    segmentUpdateStatusManager.writeLoadDetailsIntoFile(Arrays.asList(updateLists), timestamp);
                    segmentStatusManager.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(), loadDetails);
                    status = true;
                } catch (IOException e) {
                    LOGGER.error("Error while writing metadata. The metadata file path is " + carbonTablePath.getMetadataDirectoryPath());
                    status = false;
                }
            } else {
                LOGGER.error("Not able to acquire the lock.");
                status = false;
            }
        } catch (Exception e) {
            LOGGER.error("Error while updating metadata. The metadata file path is " + carbonTablePath.getMetadataDirectoryPath());
            status = false;
        } finally {
            if (updateLockStatus) {
                if (updateLock.unlock()) {
                    LOGGER.info("Unlock the segment update lock successfully.");
                } else {
                    LOGGER.error("Not able to unlock the segment update lock.");
                }
            }
            if (tableLockStatus) {
                if (statusLock.unlock()) {
                    LOGGER.info("Unlock the table status lock successfully.");
                } else {
                    LOGGER.error("Not able to unlock the table status lock.");
                }
            }
        }
    }
    return status;
}

Example 4 with ICarbonLock

use of org.apache.carbondata.core.locks.ICarbonLock in project carbondata by apache.

the class CarbonDataMergerUtil method updateLoadMetadataWithMergeStatus.

/**
   * method to update table status in case of IUD Update Delta Compaction.
   * @param loadsToMerge
   * @param metaDataFilepath
   * @param MergedLoadName
   * @param carbonLoadModel
   * @param compactionType
   * @return
   */
public static boolean updateLoadMetadataWithMergeStatus(List<LoadMetadataDetails> loadsToMerge, String metaDataFilepath, String MergedLoadName, CarbonLoadModel carbonLoadModel, long mergeLoadStartTime, CompactionType compactionType) {
    boolean tableStatusUpdationStatus = false;
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    try {
        if (carbonLock.lockWithRetries()) {
            LOGGER.info("Acquired lock for the table " + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + " for table status updation ");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
            String statusFilePath = carbonTablePath.getTableStatusFilePath();
            LoadMetadataDetails[] loadDetails = SegmentStatusManager.readLoadMetadata(metaDataFilepath);
            String mergedLoadNumber = MergedLoadName.substring(MergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER) + CarbonCommonConstants.LOAD_FOLDER.length(), MergedLoadName.length());
            long modificationOrDeletionTimeStamp = CarbonUpdateUtil.readCurrentTime();
            for (LoadMetadataDetails loadDetail : loadDetails) {
                // check if this segment is merged.
                if (loadsToMerge.contains(loadDetail)) {
                    // then need to discard the compaction process and treat it as failed compaction.
                    if (loadDetail.getLoadStatus().equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_DELETE)) {
                        LOGGER.error("Compaction is aborted as the segment " + loadDetail.getLoadName() + " is deleted after the compaction is started.");
                        return false;
                    }
                    loadDetail.setLoadStatus(CarbonCommonConstants.COMPACTED);
                    loadDetail.setModificationOrdeletionTimesStamp(modificationOrDeletionTimeStamp);
                    loadDetail.setMergedLoadName(mergedLoadNumber);
                }
            }
            // create entry for merged one.
            LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails();
            loadMetadataDetails.setPartitionCount(carbonLoadModel.getPartitionId());
            loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS);
            long loadEnddate = CarbonUpdateUtil.readCurrentTime();
            loadMetadataDetails.setLoadEndTime(loadEnddate);
            loadMetadataDetails.setLoadName(mergedLoadNumber);
            loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
            loadMetadataDetails.setPartitionCount("0");
            // if this is a major compaction then set the segment as major compaction.
            if (compactionType == CompactionType.MAJOR_COMPACTION) {
                loadMetadataDetails.setMajorCompacted("true");
            }
            List<LoadMetadataDetails> updatedDetailsList = new ArrayList<>(Arrays.asList(loadDetails));
            // put the merged folder entry
            updatedDetailsList.add(loadMetadataDetails);
            try {
                SegmentStatusManager.writeLoadDetailsIntoFile(statusFilePath, updatedDetailsList.toArray(new LoadMetadataDetails[updatedDetailsList.size()]));
                tableStatusUpdationStatus = true;
            } catch (IOException e) {
                LOGGER.error("Error while writing metadata");
                tableStatusUpdationStatus = false;
            }
        } else {
            LOGGER.error("Could not able to obtain lock for table" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + "for table status updation");
        }
    } finally {
        if (carbonLock.unlock()) {
            LOGGER.info("Table unlocked successfully after table status updation" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName());
        } else {
            LOGGER.error("Unable to unlock Table lock for table" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + " during table status updation");
        }
    }
    return tableStatusUpdationStatus;
}

Example 5 with ICarbonLock

use of org.apache.carbondata.core.locks.ICarbonLock in project carbondata by apache.

the class CarbonLoaderUtil method recordLoadMetadata.

/**
   * This API will write the load level metadata for the loadmanagement module inorder to
   * manage the load and query execution management smoothly.
   *
   * @param loadCount
   * @param loadMetadataDetails
   * @param loadModel
   * @param loadStatus
   * @param startLoadTime
   * @return boolean which determines whether status update is done or not.
   * @throws IOException
   */
public static boolean recordLoadMetadata(int loadCount, LoadMetadataDetails loadMetadataDetails, CarbonLoadModel loadModel, String loadStatus, long startLoadTime) throws IOException {
    boolean status = false;
    String metaDataFilepath = loadModel.getCarbonDataLoadSchema().getCarbonTable().getMetaDataFilepath();
    AbsoluteTableIdentifier absoluteTableIdentifier = loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String tableStatusPath = carbonTablePath.getTableStatusFilePath();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    try {
        if (carbonLock.lockWithRetries()) {
            LOGGER.info("Acquired lock for table" + loadModel.getDatabaseName() + "." + loadModel.getTableName() + " for table status updation");
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaDataFilepath);
            long loadEnddate = CarbonUpdateUtil.readCurrentTime();
            loadMetadataDetails.setLoadEndTime(loadEnddate);
            loadMetadataDetails.setLoadStatus(loadStatus);
            loadMetadataDetails.setLoadName(String.valueOf(loadCount));
            loadMetadataDetails.setLoadStartTime(startLoadTime);
            List<LoadMetadataDetails> listOfLoadFolderDetails = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
            if (null != listOfLoadFolderDetailsArray) {
                Collections.addAll(listOfLoadFolderDetails, listOfLoadFolderDetailsArray);
            }
            listOfLoadFolderDetails.add(loadMetadataDetails);
            SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetails.toArray(new LoadMetadataDetails[listOfLoadFolderDetails.size()]));
            status = true;
        } else {
            LOGGER.error("Not able to acquire the lock for Table status updation for table " + loadModel.getDatabaseName() + "." + loadModel.getTableName());
        }
    } finally {
        if (carbonLock.unlock()) {
            LOGGER.info("Table unlocked successfully after table status updation" + loadModel.getDatabaseName() + "." + loadModel.getTableName());
        } else {
            LOGGER.error("Unable to unlock Table lock for table" + loadModel.getDatabaseName() + "." + loadModel.getTableName() + " during table status updation");
        }
    }
    return status;
}