Examples with CarbonFile org.apache.carbondata.core.datastore.filesystem.CarbonFile used on opensource projects

Example 1 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonUpdateUtil method getLatestTaskIdForSegment.

public static int getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
    String segmentDirPath = tablePath.getCarbonDataDirectoryPath("0", segmentId);
    // scan all the carbondata files and get the latest task ID.
    CarbonFile segment = FileFactory.getCarbonFile(segmentDirPath, FileFactory.getFileType(segmentDirPath));
    CarbonFile[] dataFiles = segment.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
                return true;
            }
            return false;
        }
    });
    int max = 0;
    if (null != dataFiles) {
        for (CarbonFile file : dataFiles) {
            int taskNumber = Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(file.getName()).split("_")[0]);
            if (taskNumber > max) {
                max = taskNumber;
            }
        }
    }
    // return max task No
    return max;
}

Example 2 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonUpdateUtil method getLatestBlockNameForSegment.

public static String getLatestBlockNameForSegment(String segmentId, CarbonTablePath tablePath) {
    String segmentDirPath = tablePath.getCarbonDataDirectoryPath("0", segmentId);
    // scan all the carbondata files and get the latest task ID.
    CarbonFile segment = FileFactory.getCarbonFile(segmentDirPath, FileFactory.getFileType(segmentDirPath));
    CarbonFile[] dataFiles = segment.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            int max = 0;
            if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
                int taskNumber = Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(file.getName()));
                if (taskNumber >= max) {
                    return true;
                }
            }
            return false;
        }
    });
    // get the latest among the data files. highest task number will be at the last.
    return dataFiles[dataFiles.length - 1].getName();
}

Example 3 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentUpdateStatusManager method getDeleteDeltaInvalidFilesList.

/**
   *
   * @param segmentId
   * @param block
   * @param needCompleteList
   * @return
   */
public CarbonFile[] getDeleteDeltaInvalidFilesList(final String segmentId, final SegmentUpdateDetails block, final boolean needCompleteList, CarbonFile[] allSegmentFiles) {
    final long deltaStartTimestamp = getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
    List<CarbonFile> files = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    for (CarbonFile eachFile : allSegmentFiles) {
        String fileName = eachFile.getName();
        if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
            String blkName = CarbonTablePath.DataFileUtil.getBlockNameFromDeleteDeltaFile(fileName);
            // complete list of delta files of that block is returned.
            if (needCompleteList && block.getBlockName().equalsIgnoreCase(blkName)) {
                files.add(eachFile);
            }
            // invalid delete delta files only will be returned.
            long timestamp = CarbonUpdateUtil.getTimeStampAsLong(CarbonTablePath.DataFileUtil.getTimeStampFromDeleteDeltaFile(fileName));
            if (block.getBlockName().equalsIgnoreCase(blkName) && (Long.compare(timestamp, deltaStartTimestamp) < 0)) {
                files.add(eachFile);
            }
        }
    }
    return files.toArray(new CarbonFile[files.size()]);
}

Example 4 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonFooterWriterTest method createFile.

private void createFile() {
    FileFactory.FileType fileType = FileFactory.getFileType(this.filePath);
    CarbonFile carbonFile = FileFactory.getCarbonFile(this.filePath, fileType);
    carbonFile.createNewFile();
}

Example 5 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class AbstractFactDataWriter method copyCarbonDataFileToCarbonStorePath.

/**
   * This method will copy the given file to carbon store location
   *
   * @param localFileName local file name with full path
   * @throws CarbonDataWriterException
   */
protected void copyCarbonDataFileToCarbonStorePath(String localFileName) throws CarbonDataWriterException {
    long copyStartTime = System.currentTimeMillis();
    LOGGER.info("Copying " + localFileName + " --> " + dataWriterVo.getCarbonDataDirectoryPath());
    try {
        CarbonFile localCarbonFile = FileFactory.getCarbonFile(localFileName, FileFactory.getFileType(localFileName));
        String carbonFilePath = dataWriterVo.getCarbonDataDirectoryPath() + localFileName.substring(localFileName.lastIndexOf(File.separator));
        copyLocalFileToCarbonStore(carbonFilePath, localFileName, CarbonCommonConstants.BYTEBUFFER_SIZE, getMaxOfBlockAndFileSize(fileSizeInBytes, localCarbonFile.getSize()));
    } catch (IOException e) {
        throw new CarbonDataWriterException("Problem while copying file from local store to carbon store", e);
    }
    LOGGER.info("Total copy time (ms) to copy file " + localFileName + " is " + (System.currentTimeMillis() - copyStartTime));
}