Examples with CarbonFile org.apache.carbondata.core.datastore.filesystem.CarbonFile used on opensource projects

Example 41 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentFileStore method writeSegmentFile.

/**
 * Write segment file to the metadata folder of the table
 * @param tablePath table path
 * @param segmentId segment id
 * @param UUID a UUID string used to construct the segment file name
 * @return segment file name
 */
public static String writeSegmentFile(String tablePath, String segmentId, String UUID) throws IOException {
    String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
    CarbonFile segmentFolder = FileFactory.getCarbonFile(segmentPath);
    CarbonFile[] indexFiles = segmentFolder.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT);
        }
    });
    if (indexFiles != null && indexFiles.length > 0) {
        SegmentFile segmentFile = new SegmentFile();
        FolderDetails folderDetails = new FolderDetails();
        folderDetails.setRelative(true);
        folderDetails.setStatus(SegmentStatus.SUCCESS.getMessage());
        for (CarbonFile file : indexFiles) {
            folderDetails.getFiles().add(file.getName());
        }
        String segmentRelativePath = segmentPath.substring(tablePath.length(), segmentPath.length());
        segmentFile.addPath(segmentRelativePath, folderDetails);
        String segmentFileFolder = CarbonTablePath.getSegmentFilesLocation(tablePath);
        CarbonFile carbonFile = FileFactory.getCarbonFile(segmentFileFolder);
        if (!carbonFile.exists()) {
            carbonFile.mkdirs(segmentFileFolder, FileFactory.getFileType(segmentFileFolder));
        }
        String segmentFileName = genSegmentFileName(segmentId, UUID) + CarbonTablePath.SEGMENT_EXT;
        // write segment info to new file.
        writeSegmentFile(segmentFile, segmentFileFolder + File.separator + segmentFileName);
        return segmentFileName;
    }
    return null;
}

Example 42 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class BlockletDataMapIndexStore method getBlockMetaInfoMap.

private Map<String, BlockMetaInfo> getBlockMetaInfoMap(TableBlockIndexUniqueIdentifier identifier, SegmentIndexFileStore indexFileStore, Set<String> filesRead) throws IOException {
    if (identifier.getMergeIndexFileName() != null) {
        CarbonFile indexMergeFile = FileFactory.getCarbonFile(identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier.getMergeIndexFileName());
        if (indexMergeFile.exists() && !filesRead.contains(indexMergeFile.getPath())) {
            indexFileStore.readAllIIndexOfSegment(new CarbonFile[] { indexMergeFile });
            filesRead.add(indexMergeFile.getPath());
        }
    }
    if (indexFileStore.getFileData(identifier.getIndexFileName()) == null) {
        indexFileStore.readAllIIndexOfSegment(new CarbonFile[] { FileFactory.getCarbonFile(identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier.getIndexFileName()) });
    }
    DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
    Map<String, BlockMetaInfo> blockMetaInfoMap = new HashMap<>();
    List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo(identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier.getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()));
    for (DataFileFooter footer : indexInfo) {
        String blockPath = footer.getBlockInfo().getTableBlockInfo().getFilePath();
        if (FileFactory.isFileExist(blockPath)) {
            blockMetaInfoMap.put(blockPath, createBlockMetaInfo(blockPath));
        } else {
            LOGGER.warn("Skipping invalid block " + footer.getBlockInfo().getBlockUniqueName() + " The block does not exist. The block might be got deleted due to clean up post" + " update/delete operation over table.");
        }
    }
    return blockMetaInfoMap;
}

Example 43 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentIndexFileStore method readAllIIndexOfSegment.

/**
 * Read all index files and keep the cache in it.
 *
 * @param segmentFile
 * @throws IOException
 */
public void readAllIIndexOfSegment(SegmentFileStore.SegmentFile segmentFile, String tablePath, SegmentStatus status, boolean ignoreStatus) throws IOException {
    List<CarbonFile> carbonIndexFiles = new ArrayList<>();
    Set<String> indexFiles = new HashSet<>();
    if (segmentFile == null) {
        return;
    }
    for (Map.Entry<String, SegmentFileStore.FolderDetails> locations : segmentFile.getLocationMap().entrySet()) {
        String location = locations.getKey();
        if (locations.getValue().getStatus().equals(status.getMessage()) || ignoreStatus) {
            if (locations.getValue().isRelative()) {
                location = tablePath + CarbonCommonConstants.FILE_SEPARATOR + location;
            }
            String mergeFileName = locations.getValue().getMergeFileName();
            if (mergeFileName != null) {
                CarbonFile mergeFile = FileFactory.getCarbonFile(location + CarbonCommonConstants.FILE_SEPARATOR + mergeFileName);
                if (mergeFile.exists() && !indexFiles.contains(mergeFile.getAbsolutePath())) {
                    carbonIndexFiles.add(mergeFile);
                    indexFiles.add(mergeFile.getAbsolutePath());
                }
            }
            for (String indexFile : locations.getValue().getFiles()) {
                CarbonFile carbonFile = FileFactory.getCarbonFile(location + CarbonCommonConstants.FILE_SEPARATOR + indexFile);
                if (carbonFile.exists() && !indexFiles.contains(carbonFile.getAbsolutePath())) {
                    carbonIndexFiles.add(carbonFile);
                    indexFiles.add(carbonFile.getAbsolutePath());
                }
            }
        }
    }
    for (int i = 0; i < carbonIndexFiles.size(); i++) {
        if (carbonIndexFiles.get(i).getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
            readMergeFile(carbonIndexFiles.get(i).getCanonicalPath());
        } else if (carbonIndexFiles.get(i).getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
            readIndexFile(carbonIndexFiles.get(i));
        }
    }
}

Example 44 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentIndexFileStore method readMergeFile.

/**
 * Read carbonindexmerge file and update the map
 *
 * @param mergeFilePath
 * @throws IOException
 */
private void readMergeFile(String mergeFilePath) throws IOException {
    ThriftReader thriftReader = new ThriftReader(mergeFilePath);
    try {
        thriftReader.open();
        MergedBlockIndexHeader indexHeader = readMergeBlockIndexHeader(thriftReader);
        MergedBlockIndex mergedBlockIndex = readMergeBlockIndex(thriftReader);
        List<String> file_names = indexHeader.getFile_names();
        List<ByteBuffer> fileData = mergedBlockIndex.getFileData();
        CarbonFile mergeFile = FileFactory.getCarbonFile(mergeFilePath);
        assert (file_names.size() == fileData.size());
        for (int i = 0; i < file_names.size(); i++) {
            carbonIndexMap.put(file_names.get(i), fileData.get(i).array());
            carbonIndexMapWithFullPath.put(mergeFile.getParentFile().getAbsolutePath() + CarbonCommonConstants.FILE_SEPARATOR + file_names.get(i), fileData.get(i).array());
        }
    } finally {
        thriftReader.close();
    }
}

Example 45 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonLockUtil method deleteExpiredSegmentLockFiles.

/**
 * Currently the segment lock files are not deleted immediately when unlock,
 * so it needs to delete expired lock files before delete loads.
 */
public static void deleteExpiredSegmentLockFiles(CarbonTable carbonTable) {
    final long currTime = System.currentTimeMillis();
    final long segmentLockFilesPreservTime = CarbonProperties.getInstance().getSegmentLockFilesPreserveHours();
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
    String lockFilesDir = CarbonTablePath.getLockFilesDirPath(absoluteTableIdentifier.getTablePath());
    CarbonFile[] files = FileFactory.getCarbonFile(lockFilesDir).listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathName) {
            if (CarbonTablePath.isSegmentLockFilePath(pathName.getName())) {
                if ((currTime - pathName.getLastModifiedTime()) > segmentLockFilesPreservTime) {
                    return true;
                }
            }
            return false;
        }
    });
    for (CarbonFile file : files) {
        file.delete();
    }
}