Examples with SegmentFileStore org.apache.carbondata.core.metadata.SegmentFileStore used on opensource projects

Example 1 with SegmentFileStore

use of org.apache.carbondata.core.metadata.SegmentFileStore in project carbondata by apache.

the class CarbonOutputCommitter method overwritePartitions.

/**
 * Overwrite the partitions in case of overwrite query. It just updates the partition map files
 * of all segment files.
 *
 * @param loadModel
 * @return
 * @throws IOException
 */
private String overwritePartitions(CarbonLoadModel loadModel, LoadMetadataDetails newMetaEntry) throws IOException {
    CarbonTable table = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    SegmentFileStore fileStore = new SegmentFileStore(loadModel.getTablePath(), loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp() + CarbonTablePath.SEGMENT_EXT);
    List<PartitionSpec> partitionSpecs = fileStore.getPartitionSpecs();
    if (partitionSpecs != null && partitionSpecs.size() > 0) {
        List<Segment> validSegments = new SegmentStatusManager(table.getAbsoluteTableIdentifier()).getValidAndInvalidSegments().getValidSegments();
        String uniqueId = String.valueOf(System.currentTimeMillis());
        List<String> tobeUpdatedSegs = new ArrayList<>();
        List<String> tobeDeletedSegs = new ArrayList<>();
        // First drop the partitions from partition mapper files of each segment
        for (Segment segment : validSegments) {
            new SegmentFileStore(table.getTablePath(), segment.getSegmentFileName()).dropPartitions(segment, partitionSpecs, uniqueId, tobeDeletedSegs, tobeUpdatedSegs);
        }
        newMetaEntry.setUpdateStatusFileName(uniqueId);
        // Commit the removed partitions in carbon store.
        CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, false, "", Segment.toSegmentList(tobeDeletedSegs), Segment.toSegmentList(tobeUpdatedSegs));
        return uniqueId;
    }
    return null;
}

Example 2 with SegmentFileStore

use of org.apache.carbondata.core.metadata.SegmentFileStore in project carbondata by apache.

the class CarbonIndexFileMergeWriter method mergeCarbonIndexFilesOfSegment.

/**
 * Merge all the carbonindex files of segment to a  merged file
 * @param tablePath
 * @param indexFileNamesTobeAdded while merging it comsiders only these files.
 *                                If null then consider all
 * @param readFileFooterFromCarbonDataFile flag to read file footer information from carbondata
 *                                         file. This will used in case of upgrade from version
 *                                         which do not store the blocklet info to current version
 * @throws IOException
 */
private String mergeCarbonIndexFilesOfSegment(String segmentId, String tablePath, List<String> indexFileNamesTobeAdded, boolean readFileFooterFromCarbonDataFile) throws IOException {
    Segment segment = Segment.getSegment(segmentId, tablePath);
    String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
    CarbonFile[] indexFiles;
    SegmentFileStore sfs = null;
    if (segment != null && segment.getSegmentFileName() != null) {
        sfs = new SegmentFileStore(tablePath, segment.getSegmentFileName());
        List<CarbonFile> indexCarbonFiles = sfs.getIndexCarbonFiles();
        indexFiles = indexCarbonFiles.toArray(new CarbonFile[indexCarbonFiles.size()]);
    } else {
        indexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentPath);
    }
    if (isCarbonIndexFilePresent(indexFiles) || indexFileNamesTobeAdded != null) {
        if (sfs == null) {
            return writeMergeIndexFileBasedOnSegmentFolder(indexFileNamesTobeAdded, readFileFooterFromCarbonDataFile, segmentPath, indexFiles);
        } else {
            return writeMergeIndexFileBasedOnSegmentFile(segmentId, indexFileNamesTobeAdded, sfs, indexFiles);
        }
    }
    return null;
}

Example 3 with SegmentFileStore

use of org.apache.carbondata.core.metadata.SegmentFileStore in project carbondata by apache.

the class BlockletDataMapFactory method getTableBlockIndexUniqueIdentifiers.

private List<TableBlockIndexUniqueIdentifier> getTableBlockIndexUniqueIdentifiers(Segment segment) throws IOException {
    List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = segmentMap.get(segment.getSegmentNo());
    if (tableBlockIndexUniqueIdentifiers == null) {
        tableBlockIndexUniqueIdentifiers = new ArrayList<>();
        Map<String, String> indexFiles;
        if (segment.getSegmentFileName() == null) {
            String path = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo());
            indexFiles = new SegmentIndexFileStore().getIndexFilesFromSegment(path);
        } else {
            SegmentFileStore fileStore = new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
            indexFiles = fileStore.getIndexFiles();
        }
        for (Map.Entry<String, String> indexFileEntry : indexFiles.entrySet()) {
            Path indexFile = new Path(indexFileEntry.getKey());
            tableBlockIndexUniqueIdentifiers.add(new TableBlockIndexUniqueIdentifier(indexFile.getParent().toString(), indexFile.getName(), indexFileEntry.getValue(), segment.getSegmentNo()));
        }
        segmentMap.put(segment.getSegmentNo(), tableBlockIndexUniqueIdentifiers);
    }
    return tableBlockIndexUniqueIdentifiers;
}

Example 4 with SegmentFileStore

use of org.apache.carbondata.core.metadata.SegmentFileStore in project carbondata by apache.

the class BlockletDataMapFactory method toDistributable.

@Override
public List<DataMapDistributable> toDistributable(Segment segment) {
    List<DataMapDistributable> distributables = new ArrayList<>();
    try {
        CarbonFile[] carbonIndexFiles;
        if (segment.getSegmentFileName() == null) {
            carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo()));
        } else {
            SegmentFileStore fileStore = new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
            Map<String, String> indexFiles = fileStore.getIndexFiles();
            carbonIndexFiles = new CarbonFile[indexFiles.size()];
            int i = 0;
            for (String indexFile : indexFiles.keySet()) {
                carbonIndexFiles[i++] = FileFactory.getCarbonFile(indexFile);
            }
        }
        for (int i = 0; i < carbonIndexFiles.length; i++) {
            Path path = new Path(carbonIndexFiles[i].getPath());
            FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
            RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
            LocatedFileStatus fileStatus = iter.next();
            String[] location = fileStatus.getBlockLocations()[0].getHosts();
            BlockletDataMapDistributable distributable = new BlockletDataMapDistributable(path.toString());
            distributable.setLocations(location);
            distributables.add(distributable);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return distributables;
}

Example 5 with SegmentFileStore

use of org.apache.carbondata.core.metadata.SegmentFileStore in project carbondata by apache.

the class CarbonOutputCommitter method abortJob.

/**
 * Update the tablestatus as fail if any fail happens.And also clean up the temp folders if any
 * are existed.
 *
 * @param context
 * @param state
 * @throws IOException
 */
@Override
public void abortJob(JobContext context, JobStatus.State state) throws IOException {
    try {
        super.abortJob(context, state);
        CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
        CarbonLoaderUtil.updateTableStatusForFailure(loadModel);
        String segmentFileName = loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp();
        LoadMetadataDetails metadataDetail = loadModel.getCurrentLoadMetadataDetail();
        if (metadataDetail != null) {
            // will be used while cleaning.
            if (!metadataDetail.getSegmentStatus().equals(SegmentStatus.SUCCESS)) {
                String readPath = CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath()) + CarbonCommonConstants.FILE_SEPARATOR + segmentFileName + CarbonTablePath.SEGMENT_EXT;
                if (FileFactory.getCarbonFile(readPath).exists()) {
                    metadataDetail.setSegmentFile(segmentFileName + CarbonTablePath.SEGMENT_EXT);
                }
            }
        }
        // Clean the temp files
        CarbonFile segTmpFolder = FileFactory.getCarbonFile(CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath()) + CarbonCommonConstants.FILE_SEPARATOR + segmentFileName + ".tmp");
        // delete temp segment folder
        if (segTmpFolder.exists()) {
            FileFactory.deleteAllCarbonFilesOfDir(segTmpFolder);
        }
        CarbonFile segmentFilePath = FileFactory.getCarbonFile(CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath()) + CarbonCommonConstants.FILE_SEPARATOR + segmentFileName + CarbonTablePath.SEGMENT_EXT);
        // Delete the temp data folders of this job if exists
        if (segmentFilePath.exists()) {
            SegmentFileStore fileStore = new SegmentFileStore(loadModel.getTablePath(), segmentFileName + CarbonTablePath.SEGMENT_EXT);
            SegmentFileStore.removeTempFolder(fileStore.getLocationMap(), segmentFileName + ".tmp", loadModel.getTablePath());
        }
        LOGGER.error("Loading failed with job status : " + state);
    } finally {
        if (segmentLock != null) {
            segmentLock.unlock();
        }
    }
}