Examples with CarbonFile org.apache.carbondata.core.datastore.filesystem.CarbonFile used on opensource projects

Example 6 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonTablePath method getCarbonUpdatedIndexFilePath.

/**
   * Below method will be used to get the index file present in the segment folder
   * based on task id
   *
   * @param taskId      task id of the file
   * @param partitionId partition number
   * @param segmentId   segment number
   * @return full qualified carbon index path
   */
public String getCarbonUpdatedIndexFilePath(final String taskId, final String partitionId, final String segmentId) {
    String segmentDir = getSegmentDir(partitionId, segmentId);
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return file.getName().startsWith(taskId) && file.getName().endsWith(INDEX_FILE_EXT);
        }
    });
    if (files.length > 0) {
        return files[0].getAbsolutePath();
    } else {
        throw new RuntimeException("Missing Carbon Updated index file for partition[" + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId + "]");
    }
}

Example 7 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonTablePath method getCarbonDeleteDeltaFilePath.

/**
   * Below method will be used to get the index file present in the segment folder
   * based on task id
   *
   * @param taskId      task id of the file
   * @param partitionId partition number
   * @param segmentId   segment number
   * @return full qualified carbon index path
   */
public String getCarbonDeleteDeltaFilePath(final String taskId, final String partitionId, final String segmentId) {
    String segmentDir = getSegmentDir(partitionId, segmentId);
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return file.getName().startsWith(taskId) && file.getName().endsWith(DELETE_DELTA_FILE_EXT);
        }
    });
    if (files.length > 0) {
        return files[0].getAbsolutePath();
    } else {
        throw new RuntimeException("Missing Carbon delete delta file index file for partition[" + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId + "]");
    }
}

Example 8 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class ManageDictionaryAndBTree method deleteDictionaryFileAndCache.

/**
   * This method will delete the dictionary files for the given column IDs and
   * clear the dictionary cache
   *
   * @param columnSchema
   * @param carbonTableIdentifier
   * @param storePath
   */
public static void deleteDictionaryFileAndCache(final ColumnSchema columnSchema, CarbonTableIdentifier carbonTableIdentifier, String storePath) {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier);
    String metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath();
    CarbonFile metadataDir = FileFactory.getCarbonFile(metadataDirectoryPath, FileFactory.getFileType(metadataDirectoryPath));
    // sort index file is created with dictionary size appended to it. So all the files
    // with a given column ID need to be listed
    CarbonFile[] listFiles = metadataDir.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile path) {
            if (path.getName().startsWith(columnSchema.getColumnUniqueId())) {
                return true;
            }
            return false;
        }
    });
    for (CarbonFile file : listFiles) {
        // still need to be deleted
        try {
            FileFactory.deleteFile(file.getCanonicalPath(), FileFactory.getFileType(file.getCanonicalPath()));
        } catch (IOException e) {
            LOGGER.error("Failed to delete dictionary or sortIndex file for column " + columnSchema.getColumnName() + "with column ID " + columnSchema.getColumnUniqueId());
        }
    }
    // remove dictionary cache
    removeDictionaryColumnFromCache(carbonTableIdentifier, storePath, columnSchema.getColumnUniqueId());
}

Example 9 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class FileFooterValidator method setUp.

@Before
public void setUp() throws Exception {
    if (setUpIsDone) {
        return;
    }
    CarbonHiveContext.sql("CREATE CUBE validatefooter DIMENSIONS (empno Integer, empname String," + " designation String," + " doj Timestamp, workgroupcategory Integer, workgroupcategoryname String, " + "deptno Integer, deptname String, projectcode Integer, projectjoindate Timestamp," + " projectenddate Timestamp) MEASURES (attendance Integer,utilization Integer," + "salary Integer) OPTIONS (PARTITIONER [PARTITION_COUNT=1])");
    CarbonHiveContext.sql("LOAD DATA fact from './src/test/resources/data.csv' INTO CUBE validatefooter " + "PARTITIONDATA(DELIMITER ',', QUOTECHAR '\"')");
    String storePath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION);
    CarbonTableIdentifier tableIdentifier = new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "validatefooter", "1");
    String segmentPath = CarbonStorePath.getCarbonTablePath(storePath, tableIdentifier).getCarbonDataDirectoryPath("0", "0");
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    CarbonFile[] list = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
                return true;
            }
            return false;
        }
    });
    for (CarbonFile file : list) {
        String fileLocation = file.getAbsolutePath();
        CarbonFile factFile = FileFactory.getCarbonFile(fileLocation, FileFactory.getFileType(fileLocation));
        long offset = factFile.getSize() - CarbonCommonConstants.LONG_SIZE_IN_BYTE;
        FileHolder fileHolder = FileFactory.getFileHolder(FileFactory.getFileType(fileLocation));
        offset = fileHolder.readLong(fileLocation, offset);
        CarbonFooterReader metaDataReader = new CarbonFooterReader(fileLocation, offset);
        fileFooter = metaDataReader.readFooter();
    }
    setUpIsDone = true;
}

Example 10 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonLoaderUtil method deletePartialLoadDataIfExist.

public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel, final boolean isCompactionFlow) throws IOException {
    CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    String metaDataLocation = carbonTable.getMetaDataFilepath();
    final LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
    //delete folder which metadata no exist in tablestatus
    for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
        final String partitionCount = i + "";
        String partitionPath = carbonTablePath.getPartitionDir(partitionCount);
        FileType fileType = FileFactory.getFileType(partitionPath);
        if (FileFactory.isFileExist(partitionPath, fileType)) {
            CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
            CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile path) {
                    String segmentId = CarbonTablePath.DataPathUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
                    boolean found = false;
                    for (int j = 0; j < details.length; j++) {
                        if (details[j].getLoadName().equals(segmentId) && details[j].getPartitionCount().equals(partitionCount)) {
                            found = true;
                            break;
                        }
                    }
                    return !found;
                }
            });
            for (int k = 0; k < listFiles.length; k++) {
                String segmentId = CarbonTablePath.DataPathUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
                if (isCompactionFlow) {
                    if (segmentId.contains(".")) {
                        deleteStorePath(listFiles[k].getAbsolutePath());
                    }
                } else {
                    if (!segmentId.contains(".")) {
                        deleteStorePath(listFiles[k].getAbsolutePath());
                    }
                }
            }
        }
    }
}