Examples with CarbonFile org.apache.carbondata.core.datastore.filesystem.CarbonFile used on opensource projects

Example 91 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonDataProcessorUtil method renameBadRecordsFromInProgressToNormal.

/**
   * @param storeLocation
   */
public static void renameBadRecordsFromInProgressToNormal(String storeLocation) {
    // get the base store location
    String badLogStoreLocation = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC);
    badLogStoreLocation = badLogStoreLocation + File.separator + storeLocation;
    FileType fileType = FileFactory.getFileType(badLogStoreLocation);
    try {
        if (!FileFactory.isFileExist(badLogStoreLocation, fileType)) {
            return;
        }
    } catch (IOException e1) {
        LOGGER.info("bad record folder does not exist");
    }
    CarbonFile carbonFile = FileFactory.getCarbonFile(badLogStoreLocation, fileType);
    CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathname) {
            if (pathname.getName().indexOf(CarbonCommonConstants.FILE_INPROGRESS_STATUS) > -1) {
                return true;
            }
            return false;
        }
    });
    String badRecordsInProgressFileName = null;
    String changedFileName = null;
    for (CarbonFile badFiles : listFiles) {
        badRecordsInProgressFileName = badFiles.getName();
        changedFileName = badLogStoreLocation + File.separator + badRecordsInProgressFileName.substring(0, badRecordsInProgressFileName.lastIndexOf('.'));
        badFiles.renameTo(changedFileName);
        if (badFiles.exists()) {
            if (!badFiles.delete()) {
                LOGGER.error("Unable to delete File : " + badFiles.getName());
            }
        }
    }
}

Example 92 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentUpdateStatusManager method getDeleteDeltaFilesList.

/**
   * Return all delta file for a block.
   * @param segmentId
   * @param blockName
   * @return
   */
public CarbonFile[] getDeleteDeltaFilesList(final String segmentId, final String blockName) {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    for (SegmentUpdateDetails block : updateDetails) {
        if ((block.getBlockName().equalsIgnoreCase(blockName)) && (block.getSegmentName().equalsIgnoreCase(segmentId)) && !CarbonUpdateUtil.isBlockInvalid((block.getStatus()))) {
            final long deltaStartTimestamp = getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            final long deltaEndTimeStamp = getEndTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            return segDir.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile pathName) {
                    String fileName = pathName.getName();
                    if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
                        String firstPart = fileName.substring(0, fileName.indexOf('.'));
                        String blkName = firstPart.substring(0, firstPart.lastIndexOf("-"));
                        long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf("-") + 1, firstPart.length()));
                        if (blockName.equals(blkName) && (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && (Long.compare(timestamp, deltaStartTimestamp) >= 0)) {
                            return true;
                        }
                    }
                    return false;
                }
            });
        }
    }
    return null;
}

Example 93 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentUpdateStatusManager method getInvalidBlockList.

/**
   * Get the invalid tasks in that segment.
   * @param segmentId
   * @return
   */
public List<String> getInvalidBlockList(String segmentId) {
    // get the original fact file timestamp from the table status file.
    List<String> listOfInvalidBlocks = new ArrayList<String>();
    SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    LoadMetadataDetails[] segmentDetails = ssm.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
    long timestampOfOriginalFacts = 0;
    String startTimestampOfUpdate = "";
    String endTimestampOfUpdate = "";
    for (LoadMetadataDetails segment : segmentDetails) {
        // find matching segment and return timestamp.
        if (segment.getLoadName().equalsIgnoreCase(segmentId)) {
            timestampOfOriginalFacts = segment.getLoadStartTime();
            startTimestampOfUpdate = segment.getUpdateDeltaStartTimestamp();
            endTimestampOfUpdate = segment.getUpdateDeltaEndTimestamp();
        }
    }
    if (startTimestampOfUpdate.isEmpty()) {
        return listOfInvalidBlocks;
    }
    // now after getting the original fact timestamp, what ever is remaining
    // files need to cross check it with table status file.
    // filter out the fact files.
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimestampOfUpdate);
    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimestampOfUpdate);
    final Long timeStampOriginalFactFinal = timestampOfOriginalFacts;
    CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathName) {
            String fileName = pathName.getName();
            if (fileName.endsWith(CarbonCommonConstants.UPDATE_DELTA_FILE_EXT)) {
                String firstPart = fileName.substring(0, fileName.indexOf('.'));
                long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1, firstPart.length()));
                if (Long.compare(timestamp, endTimeStampFinal) <= 0 && Long.compare(timestamp, startTimeStampFinal) >= 0) {
                    return false;
                }
                if (Long.compare(timestamp, timeStampOriginalFactFinal) == 0) {
                    return false;
                }
                // take the rest of files as they are invalid.
                return true;
            }
            return false;
        }
    });
    // gather the task numbers.
    for (CarbonFile updateFiles : files) {
        listOfInvalidBlocks.add(updateFiles.getName());
    }
    return listOfInvalidBlocks;
}

Example 94 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class DeleteLoadFolders method physicalFactAndMeasureMetadataDeletion.

private static boolean physicalFactAndMeasureMetadataDeletion(String path) {
    boolean status = false;
    try {
        if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
            CarbonFile file = FileFactory.getCarbonFile(path, FileFactory.getFileType(path));
            CarbonFile[] filesToBeDeleted = file.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile file) {
                    return (CarbonTablePath.isCarbonDataFile(file.getName()) || CarbonTablePath.isCarbonIndexFile(file.getName()));
                }
            });
            //entry in metadata.
            if (filesToBeDeleted.length == 0) {
                status = true;
            } else {
                for (CarbonFile eachFile : filesToBeDeleted) {
                    if (!eachFile.delete()) {
                        LOGGER.warn("Unable to delete the file as per delete command " + eachFile.getAbsolutePath());
                        status = false;
                    } else {
                        status = true;
                    }
                }
            }
            // need to delete the complete folder.
            if (status) {
                if (!file.delete()) {
                    LOGGER.warn("Unable to delete the folder as per delete command " + file.getAbsolutePath());
                    status = false;
                }
            }
        } else {
            status = false;
        }
    } catch (IOException e) {
        LOGGER.warn("Unable to delete the file as per delete command " + path);
    }
    return status;
}

Example 95 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonDataMergerUtil method checkDeleteDeltaFilesInSeg.

/**
   * Check is the segment passed qualifies for IUD delete delta compaction or not i.e.
   * if the number of delete delta files present in the segment is more than
   * numberDeltaFilesThreshold.
   *
   * @param seg
   * @param segmentUpdateStatusManager
   * @param numberDeltaFilesThreshold
   * @return
   */
private static boolean checkDeleteDeltaFilesInSeg(String seg, SegmentUpdateStatusManager segmentUpdateStatusManager, int numberDeltaFilesThreshold) {
    Set<String> uniqueBlocks = new HashSet<String>();
    List<String> blockNameList = segmentUpdateStatusManager.getBlockNameFromSegment(seg);
    for (final String blockName : blockNameList) {
        CarbonFile[] deleteDeltaFiles = segmentUpdateStatusManager.getDeleteDeltaFilesList(seg, blockName);
        // Spill Over Blocks will choose files with unique taskID.
        for (CarbonFile blocks : deleteDeltaFiles) {
            // Get Task ID and the Timestamp from the Block name for e.g.
            // part-0-3-1481084721319.carbondata => "3-1481084721319"
            String task = CarbonTablePath.DataFileUtil.getTaskNo(blocks.getName());
            String timestamp = CarbonTablePath.DataFileUtil.getTimeStampFromDeleteDeltaFile(blocks.getName());
            String taskAndTimeStamp = task + "-" + timestamp;
            uniqueBlocks.add(taskAndTimeStamp);
        }
        if (uniqueBlocks.size() > numberDeltaFilesThreshold) {
            return true;
        }
    }
    return false;
}