Examples with LoadMetadataDetails org.apache.carbondata.core.statusmanager.LoadMetadataDetails used on opensource projects

Example 11 with LoadMetadataDetails

use of org.apache.carbondata.core.statusmanager.LoadMetadataDetails in project carbondata by apache.

the class CarbonDataMergerUtil method getValidLoadDetailsWithRetaining.

/**
   * Retain the number of segments which are to be preserved and return the remaining list of
   * segments.
   *
   * @param loadMetadataDetails
   * @param numberOfSegToBeRetained
   * @return
   */
private static List<LoadMetadataDetails> getValidLoadDetailsWithRetaining(List<LoadMetadataDetails> loadMetadataDetails, int numberOfSegToBeRetained) {
    List<LoadMetadataDetails> validList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    for (LoadMetadataDetails segment : loadMetadataDetails) {
        if (isSegmentValid(segment)) {
            validList.add(segment);
        }
    }
    // check if valid list is big enough for removing the number of seg to be retained.
    // last element
    int removingIndex = validList.size() - 1;
    for (int i = validList.size(); i > 0; i--) {
        if (numberOfSegToBeRetained == 0) {
            break;
        }
        // remove last segment
        validList.remove(removingIndex--);
        numberOfSegToBeRetained--;
    }
    return validList;
}

Example 12 with LoadMetadataDetails

use of org.apache.carbondata.core.statusmanager.LoadMetadataDetails in project carbondata by apache.

the class CarbonDataMergerUtil method updateLoadMetadataIUDUpdateDeltaMergeStatus.

/**
   * Update Both Segment Update Status and Table Status for the case of IUD Delete
   * delta compaction.
   *
   * @param loadsToMerge
   * @param metaDataFilepath
   * @param carbonLoadModel
   * @return
   */
public static boolean updateLoadMetadataIUDUpdateDeltaMergeStatus(List<LoadMetadataDetails> loadsToMerge, String metaDataFilepath, CarbonLoadModel carbonLoadModel) {
    boolean status = false;
    boolean updateLockStatus = false;
    boolean tableLockStatus = false;
    String timestamp = "" + carbonLoadModel.getFactTimeStamp();
    List<String> updatedDeltaFilesList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    // This routine updateLoadMetadataIUDCompactionMergeStatus is suppose to update
    // two files as it is only called during IUD_UPDDEL_DELTA_COMPACTION. Along with
    // Table Status Metadata file (For Update Block Compaction) it has to update the
    // Table Update Status Metadata File (For corresponding Delete Delta File).
    // As the IUD_UPDDEL_DELTA_COMPACTION going to write in the same segment therefore in
    // A) Table Update Status Metadata File (Block Level)
    //      * For each blocks which is being compacted Mark 'Compacted' as the Status.
    // B) Table Status Metadata file (Segment Level)
    //      * loadStatus won't be changed to "compacted'
    //      * UpdateDeltaStartTime and UpdateDeltaEndTime will be both set to current
    //        timestamp (which is being passed from driver)
    // First the Table Update Status Metadata File should be updated as we need to get
    // the updated blocks for the segment from Table Status Metadata Update Delta Start and
    // End Timestamp.
    // Table Update Status Metadata Update.
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(absoluteTableIdentifier);
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock();
    ICarbonLock statusLock = segmentStatusManager.getTableStatusLock();
    // Update the Compacted Blocks with Compacted Status.
    try {
        updatedDeltaFilesList = segmentUpdateStatusManager.getUpdateDeltaFiles(loadsToMerge.get(0).getLoadName().toString());
    } catch (Exception e) {
        LOGGER.error("Error while getting the Update Delta Blocks.");
        status = false;
        return status;
    }
    if (updatedDeltaFilesList.size() > 0) {
        try {
            updateLockStatus = updateLock.lockWithRetries();
            tableLockStatus = statusLock.lockWithRetries();
            List<String> blockNames = new ArrayList<>(updatedDeltaFilesList.size());
            for (String compactedBlocks : updatedDeltaFilesList) {
                // Try to BlockName
                int endIndex = compactedBlocks.lastIndexOf(File.separator);
                String blkNoExt = compactedBlocks.substring(endIndex + 1, compactedBlocks.lastIndexOf("-"));
                blockNames.add(blkNoExt);
            }
            if (updateLockStatus && tableLockStatus) {
                SegmentUpdateDetails[] updateLists = segmentUpdateStatusManager.readLoadMetadata();
                for (String compactedBlocks : blockNames) {
                    // Check is the compactedBlocks name matches with oldDetails
                    for (int i = 0; i < updateLists.length; i++) {
                        if (updateLists[i].getBlockName().equalsIgnoreCase(compactedBlocks) && !CarbonCommonConstants.COMPACTED.equalsIgnoreCase(updateLists[i].getStatus()) && !CarbonCommonConstants.MARKED_FOR_DELETE.equalsIgnoreCase(updateLists[i].getStatus())) {
                            updateLists[i].setStatus(CarbonCommonConstants.COMPACTED);
                        }
                    }
                }
                LoadMetadataDetails[] loadDetails = segmentStatusManager.readLoadMetadata(metaDataFilepath);
                for (LoadMetadataDetails loadDetail : loadDetails) {
                    if (loadsToMerge.contains(loadDetail)) {
                        loadDetail.setUpdateDeltaStartTimestamp(timestamp);
                        loadDetail.setUpdateDeltaEndTimestamp(timestamp);
                        if (loadDetail.getLoadName().equalsIgnoreCase("0")) {
                            loadDetail.setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
                        }
                    }
                }
                try {
                    segmentUpdateStatusManager.writeLoadDetailsIntoFile(Arrays.asList(updateLists), timestamp);
                    segmentStatusManager.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(), loadDetails);
                    status = true;
                } catch (IOException e) {
                    LOGGER.error("Error while writing metadata. The metadata file path is " + carbonTablePath.getMetadataDirectoryPath());
                    status = false;
                }
            } else {
                LOGGER.error("Not able to acquire the lock.");
                status = false;
            }
        } catch (Exception e) {
            LOGGER.error("Error while updating metadata. The metadata file path is " + carbonTablePath.getMetadataDirectoryPath());
            status = false;
        } finally {
            if (updateLockStatus) {
                if (updateLock.unlock()) {
                    LOGGER.info("Unlock the segment update lock successfully.");
                } else {
                    LOGGER.error("Not able to unlock the segment update lock.");
                }
            }
            if (tableLockStatus) {
                if (statusLock.unlock()) {
                    LOGGER.info("Unlock the table status lock successfully.");
                } else {
                    LOGGER.error("Not able to unlock the table status lock.");
                }
            }
        }
    }
    return status;
}

Example 13 with LoadMetadataDetails

use of org.apache.carbondata.core.statusmanager.LoadMetadataDetails in project carbondata by apache.

the class CarbonDataMergerUtil method updateLoadMetadataWithMergeStatus.

/**
   * method to update table status in case of IUD Update Delta Compaction.
   * @param loadsToMerge
   * @param metaDataFilepath
   * @param MergedLoadName
   * @param carbonLoadModel
   * @param compactionType
   * @return
   */
public static boolean updateLoadMetadataWithMergeStatus(List<LoadMetadataDetails> loadsToMerge, String metaDataFilepath, String MergedLoadName, CarbonLoadModel carbonLoadModel, long mergeLoadStartTime, CompactionType compactionType) {
    boolean tableStatusUpdationStatus = false;
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    try {
        if (carbonLock.lockWithRetries()) {
            LOGGER.info("Acquired lock for the table " + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + " for table status updation ");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
            String statusFilePath = carbonTablePath.getTableStatusFilePath();
            LoadMetadataDetails[] loadDetails = SegmentStatusManager.readLoadMetadata(metaDataFilepath);
            String mergedLoadNumber = MergedLoadName.substring(MergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER) + CarbonCommonConstants.LOAD_FOLDER.length(), MergedLoadName.length());
            long modificationOrDeletionTimeStamp = CarbonUpdateUtil.readCurrentTime();
            for (LoadMetadataDetails loadDetail : loadDetails) {
                // check if this segment is merged.
                if (loadsToMerge.contains(loadDetail)) {
                    // then need to discard the compaction process and treat it as failed compaction.
                    if (loadDetail.getLoadStatus().equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_DELETE)) {
                        LOGGER.error("Compaction is aborted as the segment " + loadDetail.getLoadName() + " is deleted after the compaction is started.");
                        return false;
                    }
                    loadDetail.setLoadStatus(CarbonCommonConstants.COMPACTED);
                    loadDetail.setModificationOrdeletionTimesStamp(modificationOrDeletionTimeStamp);
                    loadDetail.setMergedLoadName(mergedLoadNumber);
                }
            }
            // create entry for merged one.
            LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails();
            loadMetadataDetails.setPartitionCount(carbonLoadModel.getPartitionId());
            loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS);
            long loadEnddate = CarbonUpdateUtil.readCurrentTime();
            loadMetadataDetails.setLoadEndTime(loadEnddate);
            loadMetadataDetails.setLoadName(mergedLoadNumber);
            loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
            loadMetadataDetails.setPartitionCount("0");
            // if this is a major compaction then set the segment as major compaction.
            if (compactionType == CompactionType.MAJOR_COMPACTION) {
                loadMetadataDetails.setMajorCompacted("true");
            }
            List<LoadMetadataDetails> updatedDetailsList = new ArrayList<>(Arrays.asList(loadDetails));
            // put the merged folder entry
            updatedDetailsList.add(loadMetadataDetails);
            try {
                SegmentStatusManager.writeLoadDetailsIntoFile(statusFilePath, updatedDetailsList.toArray(new LoadMetadataDetails[updatedDetailsList.size()]));
                tableStatusUpdationStatus = true;
            } catch (IOException e) {
                LOGGER.error("Error while writing metadata");
                tableStatusUpdationStatus = false;
            }
        } else {
            LOGGER.error("Could not able to obtain lock for table" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + "for table status updation");
        }
    } finally {
        if (carbonLock.unlock()) {
            LOGGER.info("Table unlocked successfully after table status updation" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName());
        } else {
            LOGGER.error("Unable to unlock Table lock for table" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + " during table status updation");
        }
    }
    return tableStatusUpdationStatus;
}

Example 14 with LoadMetadataDetails

use of org.apache.carbondata.core.statusmanager.LoadMetadataDetails in project carbondata by apache.

the class DeleteLoadFolders method deleteLoadFoldersFromFileSystem.

public static boolean deleteLoadFoldersFromFileSystem(String dbName, String tableName, String storeLocation, boolean isForceDelete, LoadMetadataDetails[] details) {
    List<LoadMetadataDetails> deletedLoads = new ArrayList<LoadMetadataDetails>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    boolean isDeleted = false;
    if (details != null && details.length != 0) {
        for (LoadMetadataDetails oneLoad : details) {
            if (checkIfLoadCanBeDeleted(oneLoad, isForceDelete)) {
                String path = getSegmentPath(dbName, tableName, storeLocation, 0, oneLoad);
                boolean deletionStatus = physicalFactAndMeasureMetadataDeletion(path);
                if (deletionStatus) {
                    isDeleted = true;
                    oneLoad.setVisibility("false");
                    deletedLoads.add(oneLoad);
                    LOGGER.info("Info: Deleted the load " + oneLoad.getLoadName());
                }
            }
        }
    }
    return isDeleted;
}

Example 15 with LoadMetadataDetails

use of org.apache.carbondata.core.statusmanager.LoadMetadataDetails in project carbondata by apache.

the class CarbonLoaderUtil method deletePartialLoadDataIfExist.

public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel, final boolean isCompactionFlow) throws IOException {
    CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    String metaDataLocation = carbonTable.getMetaDataFilepath();
    final LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
    //delete folder which metadata no exist in tablestatus
    for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
        final String partitionCount = i + "";
        String partitionPath = carbonTablePath.getPartitionDir(partitionCount);
        FileType fileType = FileFactory.getFileType(partitionPath);
        if (FileFactory.isFileExist(partitionPath, fileType)) {
            CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
            CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile path) {
                    String segmentId = CarbonTablePath.DataPathUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
                    boolean found = false;
                    for (int j = 0; j < details.length; j++) {
                        if (details[j].getLoadName().equals(segmentId) && details[j].getPartitionCount().equals(partitionCount)) {
                            found = true;
                            break;
                        }
                    }
                    return !found;
                }
            });
            for (int k = 0; k < listFiles.length; k++) {
                String segmentId = CarbonTablePath.DataPathUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
                if (isCompactionFlow) {
                    if (segmentId.contains(".")) {
                        deleteStorePath(listFiles[k].getAbsolutePath());
                    }
                } else {
                    if (!segmentId.contains(".")) {
                        deleteStorePath(listFiles[k].getAbsolutePath());
                    }
                }
            }
        }
    }
}