Examples with Segment org.apache.carbondata.core.datamap.Segment used on opensource projects

Example 11 with Segment

use of org.apache.carbondata.core.datamap.Segment in project carbondata by apache.

the class CarbonIndexFileMergeWriter method mergeCarbonIndexFilesOfSegment.

/**
 * Merge all the carbonindex files of segment to a  merged file
 * @param tablePath
 * @param indexFileNamesTobeAdded while merging it comsiders only these files.
 *                                If null then consider all
 * @param readFileFooterFromCarbonDataFile flag to read file footer information from carbondata
 *                                         file. This will used in case of upgrade from version
 *                                         which do not store the blocklet info to current version
 * @throws IOException
 */
private String mergeCarbonIndexFilesOfSegment(String segmentId, String tablePath, List<String> indexFileNamesTobeAdded, boolean readFileFooterFromCarbonDataFile) throws IOException {
    Segment segment = Segment.getSegment(segmentId, tablePath);
    String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
    CarbonFile[] indexFiles;
    SegmentFileStore sfs = null;
    if (segment != null && segment.getSegmentFileName() != null) {
        sfs = new SegmentFileStore(tablePath, segment.getSegmentFileName());
        List<CarbonFile> indexCarbonFiles = sfs.getIndexCarbonFiles();
        indexFiles = indexCarbonFiles.toArray(new CarbonFile[indexCarbonFiles.size()]);
    } else {
        indexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentPath);
    }
    if (isCarbonIndexFilePresent(indexFiles) || indexFileNamesTobeAdded != null) {
        if (sfs == null) {
            return writeMergeIndexFileBasedOnSegmentFolder(indexFileNamesTobeAdded, readFileFooterFromCarbonDataFile, segmentPath, indexFiles);
        } else {
            return writeMergeIndexFileBasedOnSegmentFile(segmentId, indexFileNamesTobeAdded, sfs, indexFiles);
        }
    }
    return null;
}

Example 12 with Segment

use of org.apache.carbondata.core.datamap.Segment in project carbondata by apache.

the class CarbonLoaderUtil method addDataIndexSizeIntoMetaEntry.

/*
   * This method will add data size and index size into tablestatus for each segment. And also
   * returns the size of the segment.
   */
public static Long addDataIndexSizeIntoMetaEntry(LoadMetadataDetails loadMetadataDetails, String segmentId, CarbonTable carbonTable) throws IOException {
    Map<String, Long> dataIndexSize = CarbonUtil.getDataSizeAndIndexSize(carbonTable.getTablePath(), new Segment(segmentId, loadMetadataDetails.getSegmentFile()));
    Long dataSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE);
    loadMetadataDetails.setDataSize(String.valueOf(dataSize));
    Long indexSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE);
    loadMetadataDetails.setIndexSize(String.valueOf(indexSize));
    return dataSize + indexSize;
}

Example 13 with Segment

use of org.apache.carbondata.core.datamap.Segment in project carbondata by apache.

the class CarbonDataMergerUtil method compactBlockDeleteDeltaFiles.

/**
 * method to compact Delete Delta files in case of IUD Compaction.
 *
 * @param seg
 * @param blockName
 * @param segmentUpdateDetails
 * @param timestamp
 * @return
 * @throws IOException
 */
public static List<CarbonDataMergerUtilResult> compactBlockDeleteDeltaFiles(String seg, String blockName, CarbonTable table, SegmentUpdateDetails[] segmentUpdateDetails, Long timestamp) throws IOException {
    SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(table);
    List<CarbonDataMergerUtilResult> resultList = new ArrayList<CarbonDataMergerUtilResult>(1);
    // set the update status.
    segmentUpdateStatusManager.setUpdateStatusDetails(segmentUpdateDetails);
    CarbonFile[] deleteDeltaFiles = segmentUpdateStatusManager.getDeleteDeltaFilesList(new Segment(seg, null), blockName);
    String destFileName = blockName + "-" + timestamp.toString() + CarbonCommonConstants.DELETE_DELTA_FILE_EXT;
    String fullBlockFilePath = deleteDeltaFiles[0].getParentFile().getCanonicalPath() + CarbonCommonConstants.FILE_SEPARATOR + destFileName;
    List<String> deleteFilePathList = new ArrayList<String>();
    for (CarbonFile cFile : deleteDeltaFiles) {
        deleteFilePathList.add(cFile.getCanonicalPath());
    }
    CarbonDataMergerUtilResult blockDetails = new CarbonDataMergerUtilResult();
    blockDetails.setBlockName(blockName);
    blockDetails.setSegmentName(seg);
    blockDetails.setDeleteDeltaStartTimestamp(timestamp.toString());
    blockDetails.setDeleteDeltaEndTimestamp(timestamp.toString());
    try {
        if (startCompactionDeleteDeltaFiles(deleteFilePathList, blockName, fullBlockFilePath)) {
            blockDetails.setCompactionStatus(true);
        } else {
            blockDetails.setCompactionStatus(false);
        }
        resultList.add(blockDetails);
    } catch (IOException e) {
        LOGGER.error("Compaction of Delete Delta Files failed. The complete file path is " + fullBlockFilePath);
        throw new IOException();
    }
    return resultList;
}

Example 14 with Segment

use of org.apache.carbondata.core.datamap.Segment in project carbondata by apache.

the class CarbonDataMergerUtil method identifySegmentsToBeMergedBasedOnIUD.

/**
 * method to identify the segments qualified for merging in case of IUD Compaction.
 *
 * @param segments
 * @param carbonLoadModel
 * @return
 */
private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnIUD(List<LoadMetadataDetails> segments, CarbonLoadModel carbonLoadModel) {
    List<LoadMetadataDetails> validSegments = new ArrayList<>(segments.size());
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    int numberUpdateDeltaFilesThreshold = CarbonProperties.getInstance().getNoUpdateDeltaFilesThresholdForIUDCompaction();
    for (LoadMetadataDetails seg : segments) {
        if ((isSegmentValid(seg)) && checkUpdateDeltaFilesInSeg(new Segment(seg.getLoadName(), seg.getSegmentFile()), absoluteTableIdentifier, carbonLoadModel.getSegmentUpdateStatusManager(), numberUpdateDeltaFilesThreshold)) {
            validSegments.add(seg);
        }
    }
    return validSegments;
}

Example 15 with Segment

use of org.apache.carbondata.core.datamap.Segment in project carbondata by apache.

the class CarbonLoaderUtil method recordNewLoadMetadata.

/**
 * This API will write the load level metadata for the loadmanagement module inorder to
 * manage the load and query execution management smoothly.
 *
 * @param newMetaEntry
 * @param loadModel
 * @param uuid
 * @return boolean which determines whether status update is done or not.
 * @throws IOException
 */
public static boolean recordNewLoadMetadata(LoadMetadataDetails newMetaEntry, CarbonLoadModel loadModel, boolean loadStartEntry, boolean insertOverwrite, String uuid, List<Segment> segmentsToBeDeleted, List<Segment> segmentFilesTobeUpdated) throws IOException {
    boolean status = false;
    AbsoluteTableIdentifier identifier = loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    String metadataPath = CarbonTablePath.getMetadataPath(identifier.getTablePath());
    FileType fileType = FileFactory.getFileType(metadataPath);
    if (!FileFactory.isFileExist(metadataPath, fileType)) {
        FileFactory.mkdirs(metadataPath, fileType);
    }
    String tableStatusPath;
    if (loadModel.getCarbonDataLoadSchema().getCarbonTable().isChildDataMap() && !uuid.isEmpty()) {
        tableStatusPath = CarbonTablePath.getTableStatusFilePathWithUUID(identifier.getTablePath(), uuid);
    } else {
        tableStatusPath = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
    }
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    int retryCount = CarbonLockUtil.getLockProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK, CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK_DEFAULT);
    int maxTimeout = CarbonLockUtil.getLockProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK, CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK_DEFAULT);
    try {
        if (carbonLock.lockWithRetries(retryCount, maxTimeout)) {
            LOGGER.info("Acquired lock for table" + loadModel.getDatabaseName() + "." + loadModel.getTableName() + " for table status updation");
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(identifier.getTablePath()));
            List<LoadMetadataDetails> listOfLoadFolderDetails = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
            List<CarbonFile> staleFolders = new ArrayList<>();
            Collections.addAll(listOfLoadFolderDetails, listOfLoadFolderDetailsArray);
            // create a new segment Id if load has just begun else add the already generated Id
            if (loadStartEntry) {
                String segmentId = String.valueOf(SegmentStatusManager.createNewSegmentId(listOfLoadFolderDetailsArray));
                loadModel.setLoadMetadataDetails(listOfLoadFolderDetails);
                // If that is true then used the segment id as the load name.
                if (loadModel.getCarbonDataLoadSchema().getCarbonTable().isChildDataMap() && !loadModel.getSegmentId().isEmpty()) {
                    newMetaEntry.setLoadName(loadModel.getSegmentId());
                } else {
                    newMetaEntry.setLoadName(segmentId);
                    loadModel.setSegmentId(segmentId);
                }
                // is triggered
                for (LoadMetadataDetails entry : listOfLoadFolderDetails) {
                    if (entry.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS && SegmentStatusManager.isLoadInProgress(identifier, entry.getLoadName())) {
                        throw new RuntimeException("Already insert overwrite is in progress");
                    } else if (newMetaEntry.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS && entry.getSegmentStatus() == SegmentStatus.INSERT_IN_PROGRESS && SegmentStatusManager.isLoadInProgress(identifier, entry.getLoadName())) {
                        throw new RuntimeException("Already insert into or load is in progress");
                    }
                }
                listOfLoadFolderDetails.add(newMetaEntry);
            } else {
                newMetaEntry.setLoadName(String.valueOf(loadModel.getSegmentId()));
                // existing entry needs to be overwritten as the entry will exist with some
                // intermediate status
                int indexToOverwriteNewMetaEntry = 0;
                boolean found = false;
                for (LoadMetadataDetails entry : listOfLoadFolderDetails) {
                    if (entry.getLoadName().equals(newMetaEntry.getLoadName()) && entry.getLoadStartTime() == newMetaEntry.getLoadStartTime()) {
                        found = true;
                        break;
                    }
                    indexToOverwriteNewMetaEntry++;
                }
                if (insertOverwrite) {
                    for (LoadMetadataDetails entry : listOfLoadFolderDetails) {
                        if (entry.getSegmentStatus() != SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) {
                            entry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
                            // For insert overwrite, we will delete the old segment folder immediately
                            // So collect the old segments here
                            addToStaleFolders(identifier, staleFolders, entry);
                        }
                    }
                }
                if (!found) {
                    LOGGER.error("Entry not found to update " + newMetaEntry + " From list :: " + listOfLoadFolderDetails);
                }
                listOfLoadFolderDetails.set(indexToOverwriteNewMetaEntry, newMetaEntry);
            }
            // so empty segment folder should be deleted
            if (newMetaEntry.getSegmentStatus() == SegmentStatus.MARKED_FOR_DELETE) {
                addToStaleFolders(identifier, staleFolders, newMetaEntry);
            }
            for (LoadMetadataDetails detail : listOfLoadFolderDetails) {
                // if the segments is in the list of marked for delete then update the status.
                if (segmentsToBeDeleted.contains(new Segment(detail.getLoadName(), null))) {
                    detail.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
                } else if (segmentFilesTobeUpdated.contains(Segment.toSegment(detail.getLoadName()))) {
                    detail.setSegmentFile(detail.getLoadName() + "_" + newMetaEntry.getUpdateStatusFileName() + CarbonTablePath.SEGMENT_EXT);
                }
            }
            SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetails.toArray(new LoadMetadataDetails[listOfLoadFolderDetails.size()]));
            // Delete all old stale segment folders
            for (CarbonFile staleFolder : staleFolders) {
                // folder still remaining stale folders should be deleted
                try {
                    CarbonUtil.deleteFoldersAndFiles(staleFolder);
                } catch (IOException | InterruptedException e) {
                    LOGGER.error("Failed to delete stale folder: " + e.getMessage());
                }
            }
            status = true;
        } else {
            LOGGER.error("Not able to acquire the lock for Table status updation for table " + loadModel.getDatabaseName() + "." + loadModel.getTableName());
        }
        ;
    } finally {
        if (carbonLock.unlock()) {
            LOGGER.info("Table unlocked successfully after table status updation" + loadModel.getDatabaseName() + "." + loadModel.getTableName());
        } else {
            LOGGER.error("Unable to unlock Table lock for table" + loadModel.getDatabaseName() + "." + loadModel.getTableName() + " during table status updation");
        }
    }
    return status;
}