Examples with CarbonFileFilter org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter used on opensource projects

Example 26 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class SegmentUpdateStatusManager method getInvalidBlockList.

/**
   * Get the invalid tasks in that segment.
   * @param segmentId
   * @return
   */
public List<String> getInvalidBlockList(String segmentId) {
    // get the original fact file timestamp from the table status file.
    List<String> listOfInvalidBlocks = new ArrayList<String>();
    SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    LoadMetadataDetails[] segmentDetails = ssm.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
    long timestampOfOriginalFacts = 0;
    String startTimestampOfUpdate = "";
    String endTimestampOfUpdate = "";
    for (LoadMetadataDetails segment : segmentDetails) {
        // find matching segment and return timestamp.
        if (segment.getLoadName().equalsIgnoreCase(segmentId)) {
            timestampOfOriginalFacts = segment.getLoadStartTime();
            startTimestampOfUpdate = segment.getUpdateDeltaStartTimestamp();
            endTimestampOfUpdate = segment.getUpdateDeltaEndTimestamp();
        }
    }
    if (startTimestampOfUpdate.isEmpty()) {
        return listOfInvalidBlocks;
    }
    // now after getting the original fact timestamp, what ever is remaining
    // files need to cross check it with table status file.
    // filter out the fact files.
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimestampOfUpdate);
    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimestampOfUpdate);
    final Long timeStampOriginalFactFinal = timestampOfOriginalFacts;
    CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathName) {
            String fileName = pathName.getName();
            if (fileName.endsWith(CarbonCommonConstants.UPDATE_DELTA_FILE_EXT)) {
                String firstPart = fileName.substring(0, fileName.indexOf('.'));
                long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1, firstPart.length()));
                if (Long.compare(timestamp, endTimeStampFinal) <= 0 && Long.compare(timestamp, startTimeStampFinal) >= 0) {
                    return false;
                }
                if (Long.compare(timestamp, timeStampOriginalFactFinal) == 0) {
                    return false;
                }
                // take the rest of files as they are invalid.
                return true;
            }
            return false;
        }
    });
    // gather the task numbers.
    for (CarbonFile updateFiles : files) {
        listOfInvalidBlocks.add(updateFiles.getName());
    }
    return listOfInvalidBlocks;
}

Example 27 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class DeleteLoadFolders method physicalFactAndMeasureMetadataDeletion.

private static boolean physicalFactAndMeasureMetadataDeletion(String path) {
    boolean status = false;
    try {
        if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
            CarbonFile file = FileFactory.getCarbonFile(path, FileFactory.getFileType(path));
            CarbonFile[] filesToBeDeleted = file.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile file) {
                    return (CarbonTablePath.isCarbonDataFile(file.getName()) || CarbonTablePath.isCarbonIndexFile(file.getName()));
                }
            });
            //entry in metadata.
            if (filesToBeDeleted.length == 0) {
                status = true;
            } else {
                for (CarbonFile eachFile : filesToBeDeleted) {
                    if (!eachFile.delete()) {
                        LOGGER.warn("Unable to delete the file as per delete command " + eachFile.getAbsolutePath());
                        status = false;
                    } else {
                        status = true;
                    }
                }
            }
            // need to delete the complete folder.
            if (status) {
                if (!file.delete()) {
                    LOGGER.warn("Unable to delete the folder as per delete command " + file.getAbsolutePath());
                    status = false;
                }
            }
        } else {
            status = false;
        }
    } catch (IOException e) {
        LOGGER.warn("Unable to delete the file as per delete command " + path);
    }
    return status;
}

Example 28 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class DeleteLoadFolders method physicalFactAndMeasureMetadataDeletion.

public static void physicalFactAndMeasureMetadataDeletion(AbsoluteTableIdentifier absoluteTableIdentifier, String metadataPath, boolean isForceDelete, List<PartitionSpec> specs) {
    LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath);
    for (LoadMetadataDetails oneLoad : currentDetails) {
        if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) {
            try {
                if (oneLoad.getSegmentFile() != null) {
                    SegmentFileStore.deleteSegment(absoluteTableIdentifier.getTablePath(), oneLoad.getSegmentFile(), specs);
                } else {
                    String path = getSegmentPath(absoluteTableIdentifier, oneLoad);
                    boolean status = false;
                    if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
                        CarbonFile file = FileFactory.getCarbonFile(path, FileFactory.getFileType(path));
                        CarbonFile[] filesToBeDeleted = file.listFiles(new CarbonFileFilter() {

                            @Override
                            public boolean accept(CarbonFile file) {
                                return (CarbonTablePath.isCarbonDataFile(file.getName()) || CarbonTablePath.isCarbonIndexFile(file.getName()));
                            }
                        });
                        // entry in metadata.
                        if (filesToBeDeleted.length == 0) {
                            status = true;
                        } else {
                            for (CarbonFile eachFile : filesToBeDeleted) {
                                if (!eachFile.delete()) {
                                    LOGGER.warn("Unable to delete the file as per delete command " + eachFile.getAbsolutePath());
                                    status = false;
                                } else {
                                    status = true;
                                }
                            }
                        }
                        // need to delete the complete folder.
                        if (status) {
                            if (!file.delete()) {
                                LOGGER.warn("Unable to delete the folder as per delete command " + file.getAbsolutePath());
                            }
                        }
                    } else {
                        LOGGER.warn("Files are not found in segment " + path + " it seems, files are already being deleted");
                    }
                }
            } catch (IOException e) {
                LOGGER.warn("Unable to delete the file as per delete command " + oneLoad.getLoadName());
            }
        }
    }
}

Example 29 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class SegmentUpdateStatusManager method getDeleteDeltaFilesList.

/**
 * Return all delta file for a block.
 * @param segmentId
 * @param blockName
 * @return
 */
public CarbonFile[] getDeleteDeltaFilesList(final Segment segmentId, final String blockName) {
    String segmentPath = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId.getSegmentNo());
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    for (SegmentUpdateDetails block : updateDetails) {
        if ((block.getBlockName().equalsIgnoreCase(blockName)) && (block.getSegmentName().equalsIgnoreCase(segmentId.getSegmentNo())) && !CarbonUpdateUtil.isBlockInvalid((block.getSegmentStatus()))) {
            final long deltaStartTimestamp = getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            final long deltaEndTimeStamp = getEndTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            return segDir.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile pathName) {
                    String fileName = pathName.getName();
                    if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
                        String firstPart = fileName.substring(0, fileName.indexOf('.'));
                        String blkName = firstPart.substring(0, firstPart.lastIndexOf("-"));
                        long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf("-") + 1, firstPart.length()));
                        if (blockName.equals(blkName) && (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && (Long.compare(timestamp, deltaStartTimestamp) >= 0)) {
                            return true;
                        }
                    }
                    return false;
                }
            });
        }
    }
    return null;
}

Example 30 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class SegmentFileStore method writeSegmentFile.

/**
 * Write segment information to the segment folder with indexfilename and
 * corresponding partitions.
 */
public static void writeSegmentFile(String tablePath, final String taskNo, String location, String timeStamp, List<String> partionNames) throws IOException {
    String tempFolderLoc = timeStamp + ".tmp";
    String writePath = CarbonTablePath.getSegmentFilesLocation(tablePath) + "/" + tempFolderLoc;
    CarbonFile carbonFile = FileFactory.getCarbonFile(writePath);
    if (!carbonFile.exists()) {
        carbonFile.mkdirs(writePath, FileFactory.getFileType(writePath));
    }
    CarbonFile tempFolder = FileFactory.getCarbonFile(location + CarbonCommonConstants.FILE_SEPARATOR + tempFolderLoc);
    if (tempFolder.exists() && partionNames.size() > 0) {
        CarbonFile[] carbonFiles = tempFolder.listFiles(new CarbonFileFilter() {

            @Override
            public boolean accept(CarbonFile file) {
                return file.getName().startsWith(taskNo) && file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT);
            }
        });
        if (carbonFiles != null && carbonFiles.length > 0) {
            boolean isRelative = false;
            if (location.startsWith(tablePath)) {
                location = location.substring(tablePath.length(), location.length());
                isRelative = true;
            }
            SegmentFile segmentFile = new SegmentFile();
            FolderDetails folderDetails = new FolderDetails();
            folderDetails.setRelative(isRelative);
            folderDetails.setPartitions(partionNames);
            folderDetails.setStatus(SegmentStatus.SUCCESS.getMessage());
            for (CarbonFile file : carbonFiles) {
                folderDetails.getFiles().add(file.getName());
            }
            segmentFile.addPath(location, folderDetails);
            String path = writePath + "/" + taskNo + CarbonTablePath.SEGMENT_EXT;
            // write segment info to new file.
            writeSegmentFile(segmentFile, path);
        }
    }
}