Examples with SegmentRefreshInfo org.apache.carbondata.core.statusmanager.SegmentRefreshInfo used on opensource projects

Example 1 with SegmentRefreshInfo

use of org.apache.carbondata.core.statusmanager.SegmentRefreshInfo in project carbondata by apache.

the class TableStatusReadCommittedScope method getCommittedSegmentRefreshInfo.

public SegmentRefreshInfo getCommittedSegmentRefreshInfo(Segment segment, UpdateVO updateVo) {
    SegmentRefreshInfo segmentRefreshInfo;
    long segmentFileTimeStamp = 0L;
    String segmentFileName = segment.getSegmentFileName();
    if (null != segmentFileName) {
        // Do not use getLastModifiedTime API on segment file carbon file object as it will slow down
        // operation in Object stores like S3. Now the segment file is always written for operations
        // which was overwriting earlier, so this timestamp can be checked always to check whether
        // to refresh the cache or not
        segmentFileTimeStamp = Long.parseLong(segmentFileName.substring(segmentFileName.indexOf(CarbonCommonConstants.UNDERSCORE) + 1, segmentFileName.lastIndexOf(CarbonCommonConstants.POINT)));
    }
    if (updateVo != null) {
        segmentRefreshInfo = new SegmentRefreshInfo(updateVo.getLatestUpdateTimestamp(), 0, segmentFileTimeStamp);
    } else {
        segmentRefreshInfo = new SegmentRefreshInfo(0L, 0, segmentFileTimeStamp);
    }
    return segmentRefreshInfo;
}

Example 2 with SegmentRefreshInfo

use of org.apache.carbondata.core.statusmanager.SegmentRefreshInfo in project carbondata by apache.

the class LatestFilesReadCommittedScope method takeCarbonIndexFileSnapShot.

private void takeCarbonIndexFileSnapShot(CarbonFile[] carbonIndexFiles) {
    Map<String, List<String>> indexFileStore = new HashMap<>();
    Map<String, SegmentRefreshInfo> segmentTimestampUpdaterMap = new HashMap<>();
    for (CarbonFile carbonIndexFile : carbonIndexFiles) {
        // TODO. Nested File Paths.
        if (carbonIndexFile.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT) || carbonIndexFile.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
            // Get Segment Name from the IndexFile.
            String indexFilePath = FileFactory.getUpdatedFilePath(carbonIndexFile.getAbsolutePath());
            String timestamp = getSegmentID(carbonIndexFile.getName(), indexFilePath);
            // TODO. During Partition table handling, place Segment File Name.
            List<String> indexList;
            SegmentRefreshInfo segmentRefreshInfo;
            if (indexFileStore.get(timestamp) == null) {
                indexList = new ArrayList<>(1);
                segmentRefreshInfo = new SegmentRefreshInfo(carbonIndexFile.getLastModifiedTime(), 0, 0L);
                segmentTimestampUpdaterMap.put(timestamp, segmentRefreshInfo);
            } else {
                // Entry is already present.
                indexList = indexFileStore.get(timestamp);
                segmentRefreshInfo = segmentTimestampUpdaterMap.get(timestamp);
            }
            indexList.add(indexFilePath);
            if (segmentRefreshInfo.getSegmentUpdatedTimestamp() < carbonIndexFile.getLastModifiedTime()) {
                segmentRefreshInfo.setSegmentUpdatedTimestamp(carbonIndexFile.getLastModifiedTime());
            }
            indexFileStore.put(timestamp, indexList);
            segmentRefreshInfo.setCountOfFileInSegment(indexList.size());
        }
    }
    this.readCommittedIndexFileSnapShot = new ReadCommittedIndexFileSnapShot(indexFileStore, segmentTimestampUpdaterMap);
    prepareLoadMetadata();
}