Examples with CarbonFile org.apache.carbondata.core.datastore.filesystem.CarbonFile used on opensource projects

Example 36 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonIndexFileMergeWriter method mergeCarbonIndexFilesOfSegment.

/**
 * Merge all the carbonindex files of segment to a  merged file
 * @param tablePath
 * @param indexFileNamesTobeAdded while merging it comsiders only these files.
 *                                If null then consider all
 * @param readFileFooterFromCarbonDataFile flag to read file footer information from carbondata
 *                                         file. This will used in case of upgrade from version
 *                                         which do not store the blocklet info to current version
 * @throws IOException
 */
private String mergeCarbonIndexFilesOfSegment(String segmentId, String tablePath, List<String> indexFileNamesTobeAdded, boolean readFileFooterFromCarbonDataFile) throws IOException {
    Segment segment = Segment.getSegment(segmentId, tablePath);
    String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
    CarbonFile[] indexFiles;
    SegmentFileStore sfs = null;
    if (segment != null && segment.getSegmentFileName() != null) {
        sfs = new SegmentFileStore(tablePath, segment.getSegmentFileName());
        List<CarbonFile> indexCarbonFiles = sfs.getIndexCarbonFiles();
        indexFiles = indexCarbonFiles.toArray(new CarbonFile[indexCarbonFiles.size()]);
    } else {
        indexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentPath);
    }
    if (isCarbonIndexFilePresent(indexFiles) || indexFileNamesTobeAdded != null) {
        if (sfs == null) {
            return writeMergeIndexFileBasedOnSegmentFolder(indexFileNamesTobeAdded, readFileFooterFromCarbonDataFile, segmentPath, indexFiles);
        } else {
            return writeMergeIndexFileBasedOnSegmentFile(segmentId, indexFileNamesTobeAdded, sfs, indexFiles);
        }
    }
    return null;
}

Example 37 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonTablePath method getCarbonIndexFilePath.

/**
 * Below method will be used to get the index file present in the segment folder
 * based on task id
 *
 * @param taskId      task id of the file
 * @param segmentId   segment number
 * @return full qualified carbon index path
 */
private static String getCarbonIndexFilePath(final String tablePath, final String taskId, final String segmentId, final String bucketNumber) {
    String segmentDir = getSegmentPath(tablePath, segmentId);
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            if (bucketNumber.equals("-1")) {
                return file.getName().startsWith(taskId) && file.getName().endsWith(INDEX_FILE_EXT);
            }
            return file.getName().startsWith(taskId + "-" + bucketNumber) && file.getName().endsWith(INDEX_FILE_EXT);
        }
    });
    if (files.length > 0) {
        return files[0].getAbsolutePath();
    } else {
        throw new RuntimeException("Missing Carbon index file for Segment[" + segmentId + "], " + "taskId[" + taskId + "]");
    }
}

Example 38 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonTablePath method getActualSchemaFilePath.

private static String getActualSchemaFilePath(String tablePath) {
    String metaPath = tablePath + CarbonCommonConstants.FILE_SEPARATOR + METADATA_DIR;
    CarbonFile carbonFile = FileFactory.getCarbonFile(metaPath);
    CarbonFile[] schemaFile = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return file.getName().startsWith(SCHEMA_FILE);
        }
    });
    if (schemaFile != null && schemaFile.length > 0) {
        return schemaFile[0].getAbsolutePath();
    } else {
        return metaPath + CarbonCommonConstants.FILE_SEPARATOR + SCHEMA_FILE;
    }
}

Example 39 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class BlockletDataMapFactory method getDataMaps.

@Override
public List<CoarseGrainDataMap> getDataMaps(DataMapDistributable distributable) throws IOException {
    BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
    List<TableBlockIndexUniqueIdentifier> identifiers = new ArrayList<>();
    Path indexPath = new Path(mapDistributable.getFilePath());
    String segmentNo = mapDistributable.getSegment().getSegmentNo();
    if (indexPath.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
        String parent = indexPath.getParent().toString();
        identifiers.add(new TableBlockIndexUniqueIdentifier(parent, indexPath.getName(), null, segmentNo));
    } else if (indexPath.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
        SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
        CarbonFile carbonFile = FileFactory.getCarbonFile(indexPath.toString());
        String parentPath = carbonFile.getParentFile().getAbsolutePath();
        List<String> indexFiles = fileStore.getIndexFilesFromMergeFile(carbonFile.getAbsolutePath());
        for (String indexFile : indexFiles) {
            identifiers.add(new TableBlockIndexUniqueIdentifier(parentPath, indexFile, carbonFile.getName(), segmentNo));
        }
    }
    List<CoarseGrainDataMap> dataMaps;
    try {
        dataMaps = cache.getAll(identifiers);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return dataMaps;
}

Example 40 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentFileStore method removeTempFolder.

/**
 * Remove temp stage folder in case of job aborted.
 *
 * @param locationMap
 * @param tmpFolder
 * @param tablePath
 */
public static void removeTempFolder(Map<String, FolderDetails> locationMap, String tmpFolder, String tablePath) {
    if (locationMap == null) {
        return;
    }
    for (Map.Entry<String, SegmentFileStore.FolderDetails> entry : locationMap.entrySet()) {
        String location = entry.getKey();
        if (entry.getValue().isRelative()) {
            location = tablePath + CarbonCommonConstants.FILE_SEPARATOR + location;
        }
        CarbonFile oldFolder = FileFactory.getCarbonFile(location + CarbonCommonConstants.FILE_SEPARATOR + tmpFolder);
        if (oldFolder.exists()) {
            FileFactory.deleteAllCarbonFilesOfDir(oldFolder);
        }
    }
}