Examples with CarbonFileFilter org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter used on opensource projects

Example 1 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonUpdateUtil method getLatestTaskIdForSegment.

public static int getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
    String segmentDirPath = tablePath.getCarbonDataDirectoryPath("0", segmentId);
    // scan all the carbondata files and get the latest task ID.
    CarbonFile segment = FileFactory.getCarbonFile(segmentDirPath, FileFactory.getFileType(segmentDirPath));
    CarbonFile[] dataFiles = segment.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
                return true;
            }
            return false;
        }
    });
    int max = 0;
    if (null != dataFiles) {
        for (CarbonFile file : dataFiles) {
            int taskNumber = Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(file.getName()).split("_")[0]);
            if (taskNumber > max) {
                max = taskNumber;
            }
        }
    }
    // return max task No
    return max;
}

Example 2 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonUpdateUtil method getLatestBlockNameForSegment.

public static String getLatestBlockNameForSegment(String segmentId, CarbonTablePath tablePath) {
    String segmentDirPath = tablePath.getCarbonDataDirectoryPath("0", segmentId);
    // scan all the carbondata files and get the latest task ID.
    CarbonFile segment = FileFactory.getCarbonFile(segmentDirPath, FileFactory.getFileType(segmentDirPath));
    CarbonFile[] dataFiles = segment.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            int max = 0;
            if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
                int taskNumber = Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(file.getName()));
                if (taskNumber >= max) {
                    return true;
                }
            }
            return false;
        }
    });
    // get the latest among the data files. highest task number will be at the last.
    return dataFiles[dataFiles.length - 1].getName();
}

Example 3 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonTablePath method getCarbonUpdatedIndexFilePath.

/**
   * Below method will be used to get the index file present in the segment folder
   * based on task id
   *
   * @param taskId      task id of the file
   * @param partitionId partition number
   * @param segmentId   segment number
   * @return full qualified carbon index path
   */
public String getCarbonUpdatedIndexFilePath(final String taskId, final String partitionId, final String segmentId) {
    String segmentDir = getSegmentDir(partitionId, segmentId);
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return file.getName().startsWith(taskId) && file.getName().endsWith(INDEX_FILE_EXT);
        }
    });
    if (files.length > 0) {
        return files[0].getAbsolutePath();
    } else {
        throw new RuntimeException("Missing Carbon Updated index file for partition[" + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId + "]");
    }
}

Example 4 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonTablePath method getCarbonDeleteDeltaFilePath.

/**
   * Below method will be used to get the index file present in the segment folder
   * based on task id
   *
   * @param taskId      task id of the file
   * @param partitionId partition number
   * @param segmentId   segment number
   * @return full qualified carbon index path
   */
public String getCarbonDeleteDeltaFilePath(final String taskId, final String partitionId, final String segmentId) {
    String segmentDir = getSegmentDir(partitionId, segmentId);
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return file.getName().startsWith(taskId) && file.getName().endsWith(DELETE_DELTA_FILE_EXT);
        }
    });
    if (files.length > 0) {
        return files[0].getAbsolutePath();
    } else {
        throw new RuntimeException("Missing Carbon delete delta file index file for partition[" + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId + "]");
    }
}

Example 5 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class ManageDictionaryAndBTree method deleteDictionaryFileAndCache.

/**
   * This method will delete the dictionary files for the given column IDs and
   * clear the dictionary cache
   *
   * @param columnSchema
   * @param carbonTableIdentifier
   * @param storePath
   */
public static void deleteDictionaryFileAndCache(final ColumnSchema columnSchema, CarbonTableIdentifier carbonTableIdentifier, String storePath) {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier);
    String metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath();
    CarbonFile metadataDir = FileFactory.getCarbonFile(metadataDirectoryPath, FileFactory.getFileType(metadataDirectoryPath));
    // sort index file is created with dictionary size appended to it. So all the files
    // with a given column ID need to be listed
    CarbonFile[] listFiles = metadataDir.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile path) {
            if (path.getName().startsWith(columnSchema.getColumnUniqueId())) {
                return true;
            }
            return false;
        }
    });
    for (CarbonFile file : listFiles) {
        // still need to be deleted
        try {
            FileFactory.deleteFile(file.getCanonicalPath(), FileFactory.getFileType(file.getCanonicalPath()));
        } catch (IOException e) {
            LOGGER.error("Failed to delete dictionary or sortIndex file for column " + columnSchema.getColumnName() + "with column ID " + columnSchema.getColumnUniqueId());
        }
    }
    // remove dictionary cache
    removeDictionaryColumnFromCache(carbonTableIdentifier, storePath, columnSchema.getColumnUniqueId());
}