Examples with SegmentUpdateDetails org.apache.carbondata.core.mutate.SegmentUpdateDetails used on opensource projects

Example 1 with SegmentUpdateDetails

use of org.apache.carbondata.core.mutate.SegmentUpdateDetails in project carbondata by apache.

the class SegmentUpdateStatusManager method getTimestampForRefreshCache.

/**
   * compares passed time stamp with status file delete timestamp and
   * returns latest timestamp from status file if both are not equal
   * returns null otherwise
   *
   * @param completeBlockName
   * @param timestamp
   * @return
   */
public String getTimestampForRefreshCache(String completeBlockName, String timestamp) {
    long cacheTimestamp = 0;
    if (null != timestamp) {
        cacheTimestamp = CarbonUpdateUtil.getTimeStampAsLong(timestamp);
    }
    String blockName = CarbonTablePath.addDataPartPrefix(CarbonUpdateUtil.getBlockName(CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.BLOCK_ID)));
    String segmentId = CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.SEGMENT_ID);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray = readLoadMetadata();
    for (SegmentUpdateDetails block : listOfSegmentUpdateDetailsArray) {
        if (segmentId.equalsIgnoreCase(block.getSegmentName()) && block.getBlockName().equalsIgnoreCase(blockName) && !CarbonUpdateUtil.isBlockInvalid(block.getStatus())) {
            long deleteTimestampFromStatusFile = block.getDeleteDeltaEndTimeAsLong();
            if (Long.compare(deleteTimestampFromStatusFile, cacheTimestamp) == 0) {
                return null;
            } else {
                return block.getDeleteDeltaEndTimestamp();
            }
        }
    }
    return null;
}

Example 2 with SegmentUpdateDetails

use of org.apache.carbondata.core.mutate.SegmentUpdateDetails in project carbondata by apache.

the class SegmentUpdateStatusManager method getUpdateDeltaFilesForSegment.

/**
   * Returns all update delta files of specified Segment.
   *
   * @param segmentId
   * @param validUpdateFiles
   * @param fileExtension
   * @param excludeOriginalFact
   * @param allFilesOfSegment
   * @return
   */
public CarbonFile[] getUpdateDeltaFilesForSegment(String segmentId, final boolean validUpdateFiles, final String fileExtension, final boolean excludeOriginalFact, CarbonFile[] allFilesOfSegment) {
    String endTimeStamp = "";
    String startTimeStamp = "";
    long factTimeStamp = 0;
    for (LoadMetadataDetails eachSeg : segmentDetails) {
        if (eachSeg.getLoadName().equalsIgnoreCase(segmentId)) {
            // if the segment is found then take the start and end time stamp.
            startTimeStamp = eachSeg.getUpdateDeltaStartTimestamp();
            endTimeStamp = eachSeg.getUpdateDeltaEndTimestamp();
            factTimeStamp = eachSeg.getLoadStartTime();
        }
    }
    // if start timestamp is empty then no update delta is found. so return empty list.
    if (startTimeStamp.isEmpty()) {
        return new CarbonFile[0];
    }
    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimeStamp);
    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimeStamp);
    final long factTimeStampFinal = factTimeStamp;
    List<CarbonFile> listOfCarbonFiles = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    for (CarbonFile eachFile : allFilesOfSegment) {
        String fileName = eachFile.getName();
        if (fileName.endsWith(fileExtension)) {
            String firstPart = fileName.substring(0, fileName.indexOf('.'));
            long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1, firstPart.length()));
            if (excludeOriginalFact) {
                if (Long.compare(factTimeStampFinal, timestamp) == 0) {
                    continue;
                }
            }
            if (validUpdateFiles) {
                if (Long.compare(timestamp, endTimeStampFinal) <= 0 && Long.compare(timestamp, startTimeStampFinal) >= 0) {
                    boolean validBlock = true;
                    for (SegmentUpdateDetails blockDetails : getUpdateStatusDetails()) {
                        if (blockDetails.getActualBlockName().equalsIgnoreCase(eachFile.getName()) && CarbonUpdateUtil.isBlockInvalid(blockDetails.getStatus())) {
                            validBlock = false;
                        }
                    }
                    if (validBlock) {
                        listOfCarbonFiles.add(eachFile);
                    }
                }
            } else {
                // invalid cases.
                if (Long.compare(timestamp, startTimeStampFinal) < 0) {
                    listOfCarbonFiles.add(eachFile);
                }
            }
        }
    }
    return listOfCarbonFiles.toArray(new CarbonFile[listOfCarbonFiles.size()]);
}

Example 3 with SegmentUpdateDetails

use of org.apache.carbondata.core.mutate.SegmentUpdateDetails in project carbondata by apache.

the class SegmentUpdateStatusManager method readLoadMetadata.

/**
   * This method loads segment update details
   *
   * @return
   */
public SegmentUpdateDetails[] readLoadMetadata() {
    Gson gsonObjectToRead = new Gson();
    DataInputStream dataInputStream = null;
    BufferedReader buffReader = null;
    InputStreamReader inStream = null;
    SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray;
    // get the updated status file identifier from the table status.
    String tableUpdateStatusIdentifier = getUpdatedStatusIdentifier();
    if (null == tableUpdateStatusIdentifier) {
        return new SegmentUpdateDetails[0];
    }
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String tableUpdateStatusPath = carbonTablePath.getMetadataDirectoryPath() + CarbonCommonConstants.FILE_SEPARATOR + tableUpdateStatusIdentifier;
    AtomicFileOperations fileOperation = new AtomicFileOperationsImpl(tableUpdateStatusPath, FileFactory.getFileType(tableUpdateStatusPath));
    try {
        if (!FileFactory.isFileExist(tableUpdateStatusPath, FileFactory.getFileType(tableUpdateStatusPath))) {
            return new SegmentUpdateDetails[0];
        }
        dataInputStream = fileOperation.openForRead();
        inStream = new InputStreamReader(dataInputStream, CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT);
        buffReader = new BufferedReader(inStream);
        listOfSegmentUpdateDetailsArray = gsonObjectToRead.fromJson(buffReader, SegmentUpdateDetails[].class);
    } catch (IOException e) {
        return new SegmentUpdateDetails[0];
    } finally {
        closeStreams(buffReader, inStream, dataInputStream);
    }
    return listOfSegmentUpdateDetailsArray;
}

Example 4 with SegmentUpdateDetails

use of org.apache.carbondata.core.mutate.SegmentUpdateDetails in project carbondata by apache.

the class CarbonDataMergerUtil method updateLoadMetadataIUDUpdateDeltaMergeStatus.

/**
   * Update Both Segment Update Status and Table Status for the case of IUD Delete
   * delta compaction.
   *
   * @param loadsToMerge
   * @param metaDataFilepath
   * @param carbonLoadModel
   * @return
   */
public static boolean updateLoadMetadataIUDUpdateDeltaMergeStatus(List<LoadMetadataDetails> loadsToMerge, String metaDataFilepath, CarbonLoadModel carbonLoadModel) {
    boolean status = false;
    boolean updateLockStatus = false;
    boolean tableLockStatus = false;
    String timestamp = "" + carbonLoadModel.getFactTimeStamp();
    List<String> updatedDeltaFilesList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    // This routine updateLoadMetadataIUDCompactionMergeStatus is suppose to update
    // two files as it is only called during IUD_UPDDEL_DELTA_COMPACTION. Along with
    // Table Status Metadata file (For Update Block Compaction) it has to update the
    // Table Update Status Metadata File (For corresponding Delete Delta File).
    // As the IUD_UPDDEL_DELTA_COMPACTION going to write in the same segment therefore in
    // A) Table Update Status Metadata File (Block Level)
    //      * For each blocks which is being compacted Mark 'Compacted' as the Status.
    // B) Table Status Metadata file (Segment Level)
    //      * loadStatus won't be changed to "compacted'
    //      * UpdateDeltaStartTime and UpdateDeltaEndTime will be both set to current
    //        timestamp (which is being passed from driver)
    // First the Table Update Status Metadata File should be updated as we need to get
    // the updated blocks for the segment from Table Status Metadata Update Delta Start and
    // End Timestamp.
    // Table Update Status Metadata Update.
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(absoluteTableIdentifier);
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock();
    ICarbonLock statusLock = segmentStatusManager.getTableStatusLock();
    // Update the Compacted Blocks with Compacted Status.
    try {
        updatedDeltaFilesList = segmentUpdateStatusManager.getUpdateDeltaFiles(loadsToMerge.get(0).getLoadName().toString());
    } catch (Exception e) {
        LOGGER.error("Error while getting the Update Delta Blocks.");
        status = false;
        return status;
    }
    if (updatedDeltaFilesList.size() > 0) {
        try {
            updateLockStatus = updateLock.lockWithRetries();
            tableLockStatus = statusLock.lockWithRetries();
            List<String> blockNames = new ArrayList<>(updatedDeltaFilesList.size());
            for (String compactedBlocks : updatedDeltaFilesList) {
                // Try to BlockName
                int endIndex = compactedBlocks.lastIndexOf(File.separator);
                String blkNoExt = compactedBlocks.substring(endIndex + 1, compactedBlocks.lastIndexOf("-"));
                blockNames.add(blkNoExt);
            }
            if (updateLockStatus && tableLockStatus) {
                SegmentUpdateDetails[] updateLists = segmentUpdateStatusManager.readLoadMetadata();
                for (String compactedBlocks : blockNames) {
                    // Check is the compactedBlocks name matches with oldDetails
                    for (int i = 0; i < updateLists.length; i++) {
                        if (updateLists[i].getBlockName().equalsIgnoreCase(compactedBlocks) && !CarbonCommonConstants.COMPACTED.equalsIgnoreCase(updateLists[i].getStatus()) && !CarbonCommonConstants.MARKED_FOR_DELETE.equalsIgnoreCase(updateLists[i].getStatus())) {
                            updateLists[i].setStatus(CarbonCommonConstants.COMPACTED);
                        }
                    }
                }
                LoadMetadataDetails[] loadDetails = segmentStatusManager.readLoadMetadata(metaDataFilepath);
                for (LoadMetadataDetails loadDetail : loadDetails) {
                    if (loadsToMerge.contains(loadDetail)) {
                        loadDetail.setUpdateDeltaStartTimestamp(timestamp);
                        loadDetail.setUpdateDeltaEndTimestamp(timestamp);
                        if (loadDetail.getLoadName().equalsIgnoreCase("0")) {
                            loadDetail.setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
                        }
                    }
                }
                try {
                    segmentUpdateStatusManager.writeLoadDetailsIntoFile(Arrays.asList(updateLists), timestamp);
                    segmentStatusManager.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(), loadDetails);
                    status = true;
                } catch (IOException e) {
                    LOGGER.error("Error while writing metadata. The metadata file path is " + carbonTablePath.getMetadataDirectoryPath());
                    status = false;
                }
            } else {
                LOGGER.error("Not able to acquire the lock.");
                status = false;
            }
        } catch (Exception e) {
            LOGGER.error("Error while updating metadata. The metadata file path is " + carbonTablePath.getMetadataDirectoryPath());
            status = false;
        } finally {
            if (updateLockStatus) {
                if (updateLock.unlock()) {
                    LOGGER.info("Unlock the segment update lock successfully.");
                } else {
                    LOGGER.error("Not able to unlock the segment update lock.");
                }
            }
            if (tableLockStatus) {
                if (statusLock.unlock()) {
                    LOGGER.info("Unlock the table status lock successfully.");
                } else {
                    LOGGER.error("Not able to unlock the table status lock.");
                }
            }
        }
    }
    return status;
}

Example 5 with SegmentUpdateDetails

use of org.apache.carbondata.core.mutate.SegmentUpdateDetails in project carbondata by apache.

the class BlockLevelTraverser method getBlockRowMapping.

/**
   *
   * @param abstractIndex
   * @param blockRowMap
   * @param segId
   * @param updateStatusManager
   * @throws KeyGenException
   */
public long getBlockRowMapping(AbstractIndex abstractIndex, Map<String, Long> blockRowMap, String segId, SegmentUpdateStatusManager updateStatusManager) throws KeyGenException {
    IndexKey searchStartKey = FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties());
    DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(abstractIndex.getSegmentProperties().getEachDimColumnValueSize(), abstractIndex.getSegmentProperties().getNumberOfSortColumns(), abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns());
    DataRefNode currentBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey);
    long count = 0;
    while (currentBlock != null) {
        String blockName = ((BlockBTreeLeafNode) currentBlock).getTableBlockInfo().getFilePath();
        blockName = CarbonTablePath.getCarbonDataFileName(blockName);
        blockName = blockName + CarbonTablePath.getCarbonDataExtension();
        long rowCount = currentBlock.nodeSize();
        String key = CarbonUpdateUtil.getSegmentBlockNameKey(segId, blockName);
        // if block is invalid then dont add the count
        SegmentUpdateDetails details = updateStatusManager.getDetailsForABlock(key);
        if (null == details || !CarbonUpdateUtil.isBlockInvalid(details.getStatus())) {
            blockRowMap.put(key, rowCount);
            count++;
        }
        currentBlock = currentBlock.getNextDataRefNode();
    }
    return count;
}