Examples with SegmentUpdateDetails org.apache.carbondata.core.mutate.SegmentUpdateDetails used on opensource projects

Example 6 with SegmentUpdateDetails

use of org.apache.carbondata.core.mutate.SegmentUpdateDetails in project carbondata by apache.

the class SegmentUpdateStatusManager method getDeltaFiles.

/**
   * Returns all delta file paths of specified block
   *
   * @param tupleId
   * @param extension
   * @return
   * @throws Exception
   */
public List<String> getDeltaFiles(String tupleId, String extension) throws Exception {
    try {
        CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
        String segment = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID);
        String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment);
        String completeBlockName = CarbonTablePath.addDataPartPrefix(CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.BLOCK_ID) + CarbonCommonConstants.FACT_FILE_EXT);
        String blockPath = carbonDataDirectoryPath + CarbonCommonConstants.FILE_SEPARATOR + completeBlockName;
        CarbonFile file = FileFactory.getCarbonFile(blockPath, FileFactory.getFileType(blockPath));
        if (!file.exists()) {
            throw new Exception("Invalid tuple id " + tupleId);
        }
        String blockNameWithoutExtn = completeBlockName.substring(0, completeBlockName.indexOf('.'));
        //blockName without timestamp
        final String blockNameFromTuple = blockNameWithoutExtn.substring(0, blockNameWithoutExtn.lastIndexOf("-"));
        SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray = readLoadMetadata();
        return getDeltaFiles(file, blockNameFromTuple, listOfSegmentUpdateDetailsArray, extension, segment);
    } catch (Exception ex) {
        String errorMsg = "Invalid tuple id " + tupleId;
        LOG.error(errorMsg);
        throw new Exception(errorMsg);
    }
}

Example 7 with SegmentUpdateDetails

use of org.apache.carbondata.core.mutate.SegmentUpdateDetails in project carbondata by apache.

the class SegmentUpdateStatusManager method getDeleteDeltaFilesList.

/**
   * Return all delta file for a block.
   * @param segmentId
   * @param blockName
   * @return
   */
public CarbonFile[] getDeleteDeltaFilesList(final String segmentId, final String blockName) {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    for (SegmentUpdateDetails block : updateDetails) {
        if ((block.getBlockName().equalsIgnoreCase(blockName)) && (block.getSegmentName().equalsIgnoreCase(segmentId)) && !CarbonUpdateUtil.isBlockInvalid((block.getStatus()))) {
            final long deltaStartTimestamp = getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            final long deltaEndTimeStamp = getEndTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            return segDir.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile pathName) {
                    String fileName = pathName.getName();
                    if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
                        String firstPart = fileName.substring(0, fileName.indexOf('.'));
                        String blkName = firstPart.substring(0, firstPart.lastIndexOf("-"));
                        long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf("-") + 1, firstPart.length()));
                        if (blockName.equals(blkName) && (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && (Long.compare(timestamp, deltaStartTimestamp) >= 0)) {
                            return true;
                        }
                    }
                    return false;
                }
            });
        }
    }
    return null;
}

Example 8 with SegmentUpdateDetails

use of org.apache.carbondata.core.mutate.SegmentUpdateDetails in project carbondata by apache.

the class SegmentUpdateStatusManager method getDeltaFiles.

/**
   * Returns all delta file paths of specified block
   *
   * @param blockDir
   * @param blockNameFromTuple
   * @param listOfSegmentUpdateDetailsArray
   * @param extension
   * @return
   */
public List<String> getDeltaFiles(CarbonFile blockDir, final String blockNameFromTuple, SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray, final String extension, String segment) {
    List<String> deleteFileList = null;
    for (SegmentUpdateDetails block : listOfSegmentUpdateDetailsArray) {
        if (block.getBlockName().equalsIgnoreCase(blockNameFromTuple) && block.getSegmentName().equalsIgnoreCase(segment) && !CarbonUpdateUtil.isBlockInvalid(block.getStatus())) {
            final long deltaStartTimestamp = getStartTimeOfDeltaFile(extension, block);
            // If there is no delete delete file , then return null
            if (deltaStartTimestamp == 0) {
                return deleteFileList;
            }
            final long deltaEndTimeStamp = getEndTimeOfDeltaFile(extension, block);
            // final long deltaStartTimestamp = block.getDeleteDeltaStartTimeAsLong();
            return getFilePaths(blockDir, blockNameFromTuple, extension, deleteFileList, deltaStartTimestamp, deltaEndTimeStamp);
        }
    }
    return deleteFileList;
}

Example 9 with SegmentUpdateDetails

use of org.apache.carbondata.core.mutate.SegmentUpdateDetails in project carbondata by apache.

the class SegmentUpdateStatusManager method populateMap.

/**
   * populate the block and its details in a map.
   */
private void populateMap() {
    blockAndDetailsMap = new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    for (SegmentUpdateDetails blockDetails : updateDetails) {
        String blockIdentifier = CarbonUpdateUtil.getSegmentBlockNameKey(blockDetails.getSegmentName(), blockDetails.getActualBlockName());
        blockAndDetailsMap.put(blockIdentifier, blockDetails);
    }
}

Example 10 with SegmentUpdateDetails

use of org.apache.carbondata.core.mutate.SegmentUpdateDetails in project carbondata by apache.

the class CarbonDataMergerUtil method updateStatusFile.

public static Boolean updateStatusFile(List<CarbonDataMergerUtilResult> updateDataMergerDetailsList, CarbonTable table, String timestamp, SegmentUpdateStatusManager segmentUpdateStatusManager) {
    List<SegmentUpdateDetails> segmentUpdateDetails = new ArrayList<SegmentUpdateDetails>(updateDataMergerDetailsList.size());
    // Check the list output.
    for (CarbonDataMergerUtilResult carbonDataMergerUtilResult : updateDataMergerDetailsList) {
        if (carbonDataMergerUtilResult.getCompactionStatus()) {
            SegmentUpdateDetails tempSegmentUpdateDetails = new SegmentUpdateDetails();
            tempSegmentUpdateDetails.setSegmentName(carbonDataMergerUtilResult.getSegmentName());
            tempSegmentUpdateDetails.setBlockName(carbonDataMergerUtilResult.getBlockName());
            for (SegmentUpdateDetails origDetails : segmentUpdateStatusManager.getUpdateStatusDetails()) {
                if (origDetails.getBlockName().equalsIgnoreCase(carbonDataMergerUtilResult.getBlockName()) && origDetails.getSegmentName().equalsIgnoreCase(carbonDataMergerUtilResult.getSegmentName())) {
                    tempSegmentUpdateDetails.setDeletedRowsInBlock(origDetails.getDeletedRowsInBlock());
                    tempSegmentUpdateDetails.setStatus(origDetails.getStatus());
                    break;
                }
            }
            tempSegmentUpdateDetails.setDeleteDeltaStartTimestamp(carbonDataMergerUtilResult.getDeleteDeltaStartTimestamp());
            tempSegmentUpdateDetails.setDeleteDeltaEndTimestamp(carbonDataMergerUtilResult.getDeleteDeltaEndTimestamp());
            segmentUpdateDetails.add(tempSegmentUpdateDetails);
        } else
            return false;
    }
    CarbonUpdateUtil.updateSegmentStatus(segmentUpdateDetails, table, timestamp, true);
    // Update the Table Status.
    String metaDataFilepath = table.getMetaDataFilepath();
    AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String tableStatusPath = carbonTablePath.getTableStatusFilePath();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    boolean lockStatus = false;
    try {
        lockStatus = carbonLock.lockWithRetries();
        if (lockStatus) {
            LOGGER.info("Acquired lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " for table status updation");
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = segmentStatusManager.readLoadMetadata(metaDataFilepath);
            for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) {
                if (loadMetadata.getLoadName().equalsIgnoreCase("0")) {
                    loadMetadata.setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
                }
            }
            try {
                segmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetailsArray);
            } catch (IOException e) {
                return false;
            }
        } else {
            LOGGER.error("Not able to acquire the lock for Table status updation for table " + table.getDatabaseName() + "." + table.getFactTableName());
        }
    } finally {
        if (lockStatus) {
            if (carbonLock.unlock()) {
                LOGGER.info("Table unlocked successfully after table status updation" + table.getDatabaseName() + "." + table.getFactTableName());
            } else {
                LOGGER.error("Unable to unlock Table lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " during table status updation");
            }
        }
    }
    return true;
}