Examples with Segment org.apache.carbondata.core.index.Segment used on opensource projects

Example 6 with Segment

use of org.apache.carbondata.core.index.Segment in project carbondata by apache.

the class CarbonLoaderUtil method addDataIndexSizeIntoMetaEntry.

/*
   * This method will add data size and index size into tablestatus for each segment. And also
   * returns the size of the segment.
   */
public static Long addDataIndexSizeIntoMetaEntry(LoadMetadataDetails loadMetadataDetails, String segmentId, CarbonTable carbonTable) throws IOException {
    Map<String, Long> dataIndexSize = CarbonUtil.getDataSizeAndIndexSize(carbonTable.getTablePath(), new Segment(segmentId, loadMetadataDetails.getSegmentFile()));
    Long dataSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE);
    loadMetadataDetails.setDataSize(String.valueOf(dataSize));
    Long indexSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE);
    loadMetadataDetails.setIndexSize(String.valueOf(indexSize));
    return dataSize + indexSize;
}

Example 7 with Segment

use of org.apache.carbondata.core.index.Segment in project carbondata by apache.

the class CarbonUtilTest method testToGetSegmentString.

@Test
public void testToGetSegmentString() {
    List<Segment> list = new ArrayList<>();
    list.add(new Segment("1", null, null));
    list.add(new Segment("2", null, null));
    String segments = CarbonUtil.convertToString(list);
    assertEquals(segments, "1,2");
}

Example 8 with Segment

use of org.apache.carbondata.core.index.Segment in project carbondata by apache.

the class CarbonIndexFileMergeWriter method mergeCarbonIndexFilesOfSegment.

/**
 * Merge all the carbon index files of segment to a  merged file
 * @param tablePath
 * @param indexFileNamesTobeAdded while merging, it considers only these files.
 *                                If null, then consider all
 * @param isOldStoreIndexFilesPresent flag to read file footer information from carbondata
 *                                         file. This will used in case of upgrade from version
 *                                         which do not store the blocklet info to current version
 * @throws IOException
 */
private String mergeCarbonIndexFilesOfSegment(String segmentId, String tablePath, List<String> indexFileNamesTobeAdded, boolean isOldStoreIndexFilesPresent, String uuid, String partitionPath) {
    Segment segment = Segment.getSegment(segmentId, tablePath);
    String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
    try {
        List<CarbonFile> indexFiles = new ArrayList<>();
        SegmentFileStore sfs = null;
        if (segment != null && segment.getSegmentFileName() != null) {
            sfs = new SegmentFileStore(tablePath, segment.getSegmentFileName());
            List<CarbonFile> indexCarbonFiles = sfs.getIndexCarbonFiles();
            if (table.isHivePartitionTable()) {
                // in case of partition table, merge index files of a partition
                List<CarbonFile> indexFilesInPartition = new ArrayList<>();
                for (CarbonFile indexCarbonFile : indexCarbonFiles) {
                    if (FileFactory.getUpdatedFilePath(indexCarbonFile.getParentFile().getPath()).equals(partitionPath)) {
                        indexFilesInPartition.add(indexCarbonFile);
                    }
                }
                indexFiles = indexFilesInPartition;
            } else {
                indexFiles = indexCarbonFiles;
            }
        }
        if (sfs == null || indexFiles.isEmpty()) {
            if (table.isHivePartitionTable()) {
                segmentPath = partitionPath;
            }
            return writeMergeIndexFileBasedOnSegmentFolder(indexFileNamesTobeAdded, isOldStoreIndexFilesPresent, segmentPath, segmentId, uuid, true);
        } else {
            return writeMergeIndexFileBasedOnSegmentFile(segmentId, indexFileNamesTobeAdded, isOldStoreIndexFilesPresent, sfs, indexFiles.toArray(new CarbonFile[0]), uuid, partitionPath);
        }
    } catch (Exception e) {
        String message = "Failed to merge index files in path: " + segmentPath + ". " + e.getMessage();
        LOGGER.error(message);
        throw new RuntimeException(message, e);
    }
}

Example 9 with Segment

use of org.apache.carbondata.core.index.Segment in project carbondata by apache.

the class CarbonDataMergerUtil method compactBlockDeleteDeltaFiles.

/**
 * method to compact Delete Delta files in case of IUD Compaction.
 *
 * @param seg
 * @param blockName
 * @param segmentUpdateDetails
 * @param timestamp
 * @return
 * @throws IOException
 */
public static List<CarbonDataMergerUtilResult> compactBlockDeleteDeltaFiles(String seg, String blockName, CarbonTable table, SegmentUpdateDetails[] segmentUpdateDetails, Long timestamp) throws IOException {
    SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(table);
    List<CarbonDataMergerUtilResult> resultList = new ArrayList<CarbonDataMergerUtilResult>(1);
    // set the update status.
    segmentUpdateStatusManager.setUpdateStatusDetails(segmentUpdateDetails);
    List<String> deleteFilePathList = segmentUpdateStatusManager.getDeleteDeltaFilesList(new Segment(seg), blockName);
    String destFileName = blockName + "-" + timestamp.toString() + CarbonCommonConstants.DELETE_DELTA_FILE_EXT;
    if (deleteFilePathList.size() > 0) {
        String deleteDeltaFilePath = deleteFilePathList.get(0);
        String fullBlockFilePath = deleteDeltaFilePath.substring(0, deleteDeltaFilePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR)) + CarbonCommonConstants.FILE_SEPARATOR + destFileName;
        CarbonDataMergerUtilResult blockDetails = new CarbonDataMergerUtilResult();
        blockDetails.setBlockName(blockName);
        blockDetails.setSegmentName(seg);
        blockDetails.setDeleteDeltaStartTimestamp(timestamp.toString());
        blockDetails.setDeleteDeltaEndTimestamp(timestamp.toString());
        try {
            startCompactionDeleteDeltaFiles(deleteFilePathList, blockName, fullBlockFilePath);
            blockDetails.setCompactionStatus(true);
            resultList.add(blockDetails);
        } catch (IOException e) {
            LOGGER.error("Compaction of Delete Delta Files failed. The complete file path is " + fullBlockFilePath);
            throw new IOException();
        }
    }
    return resultList;
}

Example 10 with Segment

use of org.apache.carbondata.core.index.Segment in project carbondata by apache.

the class IndexWriterListener method register.

/**
 * Register a IndexWriter
 */
private void register(IndexFactory factory, String segmentId, String taskNo, SegmentProperties segmentProperties) {
    assert (factory != null);
    assert (segmentId != null);
    IndexMeta meta = factory.getMeta();
    if (meta == null) {
        // if index does not have meta, no need to register
        return;
    }
    List<CarbonColumn> columns = factory.getMeta().getIndexedColumns();
    List<IndexWriter> writers = registry.get(columns);
    IndexWriter writer = null;
    try {
        writer = factory.createWriter(new Segment(segmentId), taskNo, segmentProperties);
    } catch (IOException e) {
        LOG.error("Failed to create IndexWriter: " + e.getMessage(), e);
        throw new IndexWriterException(e);
    }
    if (writers != null) {
        writers.add(writer);
    } else {
        writers = new ArrayList<>();
        writers.add(writer);
        registry.put(columns, writers);
    }
    LOG.info("IndexWriter " + writer + " added");
}