Examples with CarbonFile org.apache.carbondata.core.datastore.filesystem.CarbonFile used on opensource projects

Example 11 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonLoaderUtil method isValidSegment.

/**
   * the method returns true if the segment has carbondata file else returns false.
   *
   * @param loadModel
   * @param currentLoad
   * @return
   */
public static boolean isValidSegment(CarbonLoadModel loadModel, int currentLoad) {
    CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
    int fileCount = 0;
    int partitionCount = carbonTable.getPartitionCount();
    for (int i = 0; i < partitionCount; i++) {
        String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(i + "", currentLoad + "");
        CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
        CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

            @Override
            public boolean accept(CarbonFile file) {
                return file.getName().endsWith(CarbonTablePath.getCarbonIndexExtension()) || file.getName().endsWith(CarbonTablePath.getCarbonDataExtension());
            }
        });
        fileCount += files.length;
        if (files.length > 0) {
            return true;
        }
    }
    if (fileCount == 0) {
        return false;
    }
    return true;
}

Example 12 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonLoaderUtil method deleteStorePath.

private static void deleteStorePath(String path) {
    try {
        FileType fileType = FileFactory.getFileType(path);
        if (FileFactory.isFileExist(path, fileType)) {
            CarbonFile carbonFile = FileFactory.getCarbonFile(path, fileType);
            CarbonUtil.deleteFoldersAndFiles(carbonFile);
        }
    } catch (IOException | InterruptedException e) {
        LOGGER.error("Unable to delete the given path :: " + e.getMessage());
    }
}

Example 13 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonDataMergerUtil method checkUpdateDeltaFilesInSeg.

/**
   * This method traverses Update Delta Files inside the seg and return true
   * if UpdateDelta Files are more than IUD Compaction threshold.
   *
   * @param seg
   * @param absoluteTableIdentifier
   * @param segmentUpdateStatusManager
   * @param numberDeltaFilesThreshold
   * @return
   */
public static Boolean checkUpdateDeltaFilesInSeg(String seg, AbsoluteTableIdentifier absoluteTableIdentifier, SegmentUpdateStatusManager segmentUpdateStatusManager, int numberDeltaFilesThreshold) {
    CarbonFile[] updateDeltaFiles = null;
    Set<String> uniqueBlocks = new HashSet<String>();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", seg);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    CarbonFile[] allSegmentFiles = segDir.listFiles();
    updateDeltaFiles = segmentUpdateStatusManager.getUpdateDeltaFilesForSegment(seg, true, CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, false, allSegmentFiles);
    if (updateDeltaFiles == null) {
        return false;
    }
    // Spill Over Blocks will choose files with unique taskID.
    for (CarbonFile blocks : updateDeltaFiles) {
        // Get Task ID and the Timestamp from the Block name for e.g.
        // part-0-3-1481084721319.carbondata => "3-1481084721319"
        String task = CarbonTablePath.DataFileUtil.getTaskNo(blocks.getName());
        String timestamp = CarbonTablePath.DataFileUtil.getTimeStampFromDeleteDeltaFile(blocks.getName());
        String taskAndTimeStamp = task + "-" + timestamp;
        uniqueBlocks.add(taskAndTimeStamp);
    }
    if (uniqueBlocks.size() > numberDeltaFilesThreshold) {
        return true;
    } else {
        return false;
    }
}

Example 14 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonDataMergerUtil method getSizeOfSegment.

/**
   * For calculating the size of the specified segment
   * @param storeLocation
   * @param tableIdentifier
   * @param segId
   * @return
   */
private static long getSizeOfSegment(String storeLocation, CarbonTableIdentifier tableIdentifier, String segId) {
    String loadPath = getStoreLocation(storeLocation, tableIdentifier, segId);
    CarbonFile segmentFolder = FileFactory.getCarbonFile(loadPath, FileFactory.getFileType(loadPath));
    return getSizeOfFactFileInLoad(segmentFolder);
}

Example 15 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonUpdateUtil method getLatestTaskIdForSegment.

public static long getLatestTaskIdForSegment(String segmentId, String tablePath) {
    String segmentDirPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
    // scan all the carbondata files and get the latest task ID.
    CarbonFile segment = FileFactory.getCarbonFile(segmentDirPath, FileFactory.getFileType(segmentDirPath));
    CarbonFile[] dataFiles = segment.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
                return true;
            }
            return false;
        }
    });
    long max = 0;
    if (null != dataFiles) {
        for (CarbonFile file : dataFiles) {
            long taskNumber = Long.parseLong(CarbonTablePath.DataFileUtil.getTaskNo(file.getName()).split("_")[0]);
            if (taskNumber > max) {
                max = taskNumber;
            }
        }
    }
    // return max task No
    return max;
}