Examples with CarbonFileFilter org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter used on opensource projects

Example 6 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonTablePath method getCarbonUpdatedIndexFilePath.

/**
   * Below method will be used to get the index file present in the segment folder
   * based on task id
   *
   * @param taskId      task id of the file
   * @param partitionId partition number
   * @param segmentId   segment number
   * @return full qualified carbon index path
   */
public String getCarbonUpdatedIndexFilePath(final String taskId, final String partitionId, final String segmentId) {
    String segmentDir = getSegmentDir(partitionId, segmentId);
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return file.getName().startsWith(taskId) && file.getName().endsWith(INDEX_FILE_EXT);
        }
    });
    if (files.length > 0) {
        return files[0].getAbsolutePath();
    } else {
        throw new RuntimeException("Missing Carbon Updated index file for partition[" + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId + "]");
    }
}

Example 7 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonTablePath method getCarbonDeleteDeltaFilePath.

/**
   * Below method will be used to get the index file present in the segment folder
   * based on task id
   *
   * @param taskId      task id of the file
   * @param partitionId partition number
   * @param segmentId   segment number
   * @return full qualified carbon index path
   */
public String getCarbonDeleteDeltaFilePath(final String taskId, final String partitionId, final String segmentId) {
    String segmentDir = getSegmentDir(partitionId, segmentId);
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return file.getName().startsWith(taskId) && file.getName().endsWith(DELETE_DELTA_FILE_EXT);
        }
    });
    if (files.length > 0) {
        return files[0].getAbsolutePath();
    } else {
        throw new RuntimeException("Missing Carbon delete delta file index file for partition[" + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId + "]");
    }
}

Example 8 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonTablePath method getCarbonIndexFilePath.

/**
   * Below method will be used to get the index file present in the segment folder
   * based on task id
   *
   * @param taskId      task id of the file
   * @param partitionId partition number
   * @param segmentId   segment number
   * @return full qualified carbon index path
   */
public String getCarbonIndexFilePath(final String taskId, final String partitionId, final String segmentId, final String bucketNumber) {
    String segmentDir = getSegmentDir(partitionId, segmentId);
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            if (bucketNumber.equals("-1")) {
                return file.getName().startsWith(taskId) && file.getName().endsWith(INDEX_FILE_EXT);
            }
            return file.getName().startsWith(taskId + "-" + bucketNumber) && file.getName().endsWith(INDEX_FILE_EXT);
        }
    });
    if (files.length > 0) {
        return files[0].getAbsolutePath();
    } else {
        throw new RuntimeException("Missing Carbon index file for partition[" + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId + "]");
    }
}

Example 9 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class ManageDictionaryAndBTree method deleteDictionaryFileAndCache.

/**
   * This method will delete the dictionary files for the given column IDs and
   * clear the dictionary cache
   *
   * @param columnSchema
   * @param carbonTableIdentifier
   * @param storePath
   */
public static void deleteDictionaryFileAndCache(final ColumnSchema columnSchema, CarbonTableIdentifier carbonTableIdentifier, String storePath) {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier);
    String metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath();
    CarbonFile metadataDir = FileFactory.getCarbonFile(metadataDirectoryPath, FileFactory.getFileType(metadataDirectoryPath));
    // sort index file is created with dictionary size appended to it. So all the files
    // with a given column ID need to be listed
    CarbonFile[] listFiles = metadataDir.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile path) {
            if (path.getName().startsWith(columnSchema.getColumnUniqueId())) {
                return true;
            }
            return false;
        }
    });
    for (CarbonFile file : listFiles) {
        // still need to be deleted
        try {
            FileFactory.deleteFile(file.getCanonicalPath(), FileFactory.getFileType(file.getCanonicalPath()));
        } catch (IOException e) {
            LOGGER.error("Failed to delete dictionary or sortIndex file for column " + columnSchema.getColumnName() + "with column ID " + columnSchema.getColumnUniqueId());
        }
    }
    // remove dictionary cache
    removeDictionaryColumnFromCache(carbonTableIdentifier, storePath, columnSchema.getColumnUniqueId());
}

Example 10 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class FileFooterValidator method setUp.

@Before
public void setUp() throws Exception {
    if (setUpIsDone) {
        return;
    }
    CarbonHiveContext.sql("CREATE CUBE validatefooter DIMENSIONS (empno Integer, empname String," + " designation String," + " doj Timestamp, workgroupcategory Integer, workgroupcategoryname String, " + "deptno Integer, deptname String, projectcode Integer, projectjoindate Timestamp," + " projectenddate Timestamp) MEASURES (attendance Integer,utilization Integer," + "salary Integer) OPTIONS (PARTITIONER [PARTITION_COUNT=1])");
    CarbonHiveContext.sql("LOAD DATA fact from './src/test/resources/data.csv' INTO CUBE validatefooter " + "PARTITIONDATA(DELIMITER ',', QUOTECHAR '\"')");
    String storePath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION);
    CarbonTableIdentifier tableIdentifier = new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "validatefooter", "1");
    String segmentPath = CarbonStorePath.getCarbonTablePath(storePath, tableIdentifier).getCarbonDataDirectoryPath("0", "0");
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    CarbonFile[] list = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
                return true;
            }
            return false;
        }
    });
    for (CarbonFile file : list) {
        String fileLocation = file.getAbsolutePath();
        CarbonFile factFile = FileFactory.getCarbonFile(fileLocation, FileFactory.getFileType(fileLocation));
        long offset = factFile.getSize() - CarbonCommonConstants.LONG_SIZE_IN_BYTE;
        FileHolder fileHolder = FileFactory.getFileHolder(FileFactory.getFileType(fileLocation));
        offset = fileHolder.readLong(fileLocation, offset);
        CarbonFooterReader metaDataReader = new CarbonFooterReader(fileLocation, offset);
        fileFooter = metaDataReader.readFooter();
    }
    setUpIsDone = true;
}