Examples with CarbonFileFilter org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter used on opensource projects

Example 21 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonBadRecordUtil method renameBadRecordsFromInProgressToNormal.

/**
 * @param configuration
 * @param storeLocation
 */
private static void renameBadRecordsFromInProgressToNormal(CarbonDataLoadConfiguration configuration, String storeLocation) {
    // get the base store location
    String badLogStoreLocation = (String) configuration.getDataLoadProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH);
    if (null == badLogStoreLocation) {
        badLogStoreLocation = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC);
    }
    badLogStoreLocation = badLogStoreLocation + File.separator + storeLocation;
    FileFactory.FileType fileType = FileFactory.getFileType(badLogStoreLocation);
    try {
        if (!FileFactory.isFileExist(badLogStoreLocation, fileType)) {
            return;
        }
    } catch (IOException e1) {
        LOGGER.info("bad record folder does not exist");
    }
    CarbonFile carbonFile = FileFactory.getCarbonFile(badLogStoreLocation, fileType);
    CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathname) {
            if (pathname.getName().indexOf(CarbonCommonConstants.FILE_INPROGRESS_STATUS) > -1) {
                return true;
            }
            return false;
        }
    });
    String badRecordsInProgressFileName = null;
    String changedFileName = null;
    for (CarbonFile badFiles : listFiles) {
        badRecordsInProgressFileName = badFiles.getName();
        changedFileName = badLogStoreLocation + File.separator + badRecordsInProgressFileName.substring(0, badRecordsInProgressFileName.lastIndexOf('.'));
        badFiles.renameTo(changedFileName);
        if (badFiles.exists()) {
            if (!badFiles.delete()) {
                LOGGER.error("Unable to delete File : " + badFiles.getName());
            }
        }
    }
}

Example 22 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonLoaderUtil method isValidSegment.

/**
 * the method returns true if the segment has carbondata file else returns false.
 *
 * @param loadModel
 * @param currentLoad
 * @return
 */
public static boolean isValidSegment(CarbonLoadModel loadModel, int currentLoad) {
    int fileCount = 0;
    String segmentPath = CarbonTablePath.getSegmentPath(loadModel.getTablePath(), currentLoad + "");
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return file.getName().endsWith(CarbonTablePath.getCarbonIndexExtension()) || file.getName().endsWith(CarbonTablePath.getCarbonDataExtension());
        }
    });
    fileCount += files.length;
    if (files.length > 0) {
        return true;
    }
    if (fileCount == 0) {
        return false;
    }
    return true;
}

Example 23 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonDataMergerUtil method getSizeOfFactFileInLoad.

/**
 * Returns the size of all the carbondata files present in the segment.
 * @param carbonFile
 * @return
 */
private static long getSizeOfFactFileInLoad(CarbonFile carbonFile) {
    long factSize = 0;
    // carbon data file case.
    CarbonFile[] factFile = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return CarbonTablePath.isCarbonDataFile(file.getName());
        }
    });
    for (CarbonFile fact : factFile) {
        factSize += fact.getSize();
    }
    return factSize;
}

Example 24 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonDataProcessorUtil method renameBadRecordsFromInProgressToNormal.

/**
   * @param storeLocation
   */
public static void renameBadRecordsFromInProgressToNormal(String storeLocation) {
    // get the base store location
    String badLogStoreLocation = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC);
    badLogStoreLocation = badLogStoreLocation + File.separator + storeLocation;
    FileType fileType = FileFactory.getFileType(badLogStoreLocation);
    try {
        if (!FileFactory.isFileExist(badLogStoreLocation, fileType)) {
            return;
        }
    } catch (IOException e1) {
        LOGGER.info("bad record folder does not exist");
    }
    CarbonFile carbonFile = FileFactory.getCarbonFile(badLogStoreLocation, fileType);
    CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathname) {
            if (pathname.getName().indexOf(CarbonCommonConstants.FILE_INPROGRESS_STATUS) > -1) {
                return true;
            }
            return false;
        }
    });
    String badRecordsInProgressFileName = null;
    String changedFileName = null;
    for (CarbonFile badFiles : listFiles) {
        badRecordsInProgressFileName = badFiles.getName();
        changedFileName = badLogStoreLocation + File.separator + badRecordsInProgressFileName.substring(0, badRecordsInProgressFileName.lastIndexOf('.'));
        badFiles.renameTo(changedFileName);
        if (badFiles.exists()) {
            if (!badFiles.delete()) {
                LOGGER.error("Unable to delete File : " + badFiles.getName());
            }
        }
    }
}

Example 25 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class SegmentUpdateStatusManager method getDeleteDeltaFilesList.

/**
   * Return all delta file for a block.
   * @param segmentId
   * @param blockName
   * @return
   */
public CarbonFile[] getDeleteDeltaFilesList(final String segmentId, final String blockName) {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    for (SegmentUpdateDetails block : updateDetails) {
        if ((block.getBlockName().equalsIgnoreCase(blockName)) && (block.getSegmentName().equalsIgnoreCase(segmentId)) && !CarbonUpdateUtil.isBlockInvalid((block.getStatus()))) {
            final long deltaStartTimestamp = getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            final long deltaEndTimeStamp = getEndTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            return segDir.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile pathName) {
                    String fileName = pathName.getName();
                    if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
                        String firstPart = fileName.substring(0, fileName.indexOf('.'));
                        String blkName = firstPart.substring(0, firstPart.lastIndexOf("-"));
                        long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf("-") + 1, firstPart.length()));
                        if (blockName.equals(blkName) && (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && (Long.compare(timestamp, deltaStartTimestamp) >= 0)) {
                            return true;
                        }
                    }
                    return false;
                }
            });
        }
    }
    return null;
}