Examples with CarbonFile org.apache.carbondata.core.datastore.filesystem.CarbonFile used on opensource projects

Example 26 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonTableReader method updateSchemaTables.

/**
 * Find all the tables under the schema store path (this.carbonFileList)
 * and cache all the table names in this.tableList. Notice that whenever this method
 * is called, it clears this.tableList and populate the list by reading the files.
 */
private void updateSchemaTables(SchemaTableName schemaTableName) {
    // update logic determine later
    boolean isKeyExists = carbonCache.get().containsKey(schemaTableName);
    if (carbonFileList == null) {
        updateSchemaList();
    }
    try {
        if (isKeyExists && !FileFactory.isFileExist(CarbonTablePath.getSchemaFilePath(carbonCache.get().get(schemaTableName).carbonTable.getTablePath()), fileType)) {
            removeTableFromCache(schemaTableName);
            throw new TableNotFoundException(schemaTableName);
        }
    } catch (IOException e) {
        throw new RuntimeException();
    }
    if (isKeyExists) {
        CarbonTableCacheModel ctcm = carbonCache.get().get(schemaTableName);
        if (ctcm != null && ctcm.carbonTable.getTableInfo() != null) {
            Long latestTime = FileFactory.getCarbonFile(CarbonTablePath.getSchemaFilePath(carbonCache.get().get(schemaTableName).carbonTable.getTablePath())).getLastModifiedTime();
            Long oldTime = ctcm.carbonTable.getTableInfo().getLastUpdatedTime();
            if (DateUtils.truncate(new Date(latestTime), Calendar.MINUTE).after(DateUtils.truncate(new Date(oldTime), Calendar.MINUTE))) {
                removeTableFromCache(schemaTableName);
            }
        }
    }
    if (!tableList.contains(schemaTableName)) {
        for (CarbonFile cf : carbonFileList.listFiles()) {
            if (!cf.getName().endsWith(".mdt")) {
                for (CarbonFile table : cf.listFiles()) {
                    tableList.add(new SchemaTableName(cf.getName(), table.getName()));
                }
            }
        }
    }
}

Example 27 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class TableProcessingOperations method deletePartialLoadDataIfExist.

/**
 * @param carbonTable
 * @param isCompactionFlow
 * @throws IOException
 */
public static void deletePartialLoadDataIfExist(CarbonTable carbonTable, final boolean isCompactionFlow) throws IOException {
    String metaDataLocation = carbonTable.getMetadataPath();
    final LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation);
    // delete folder which metadata no exist in tablestatus
    String partitionPath = CarbonTablePath.getPartitionDir(carbonTable.getTablePath());
    FileFactory.FileType fileType = FileFactory.getFileType(partitionPath);
    if (FileFactory.isFileExist(partitionPath, fileType)) {
        CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
        CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

            @Override
            public boolean accept(CarbonFile path) {
                String segmentId = CarbonTablePath.DataFileUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
                boolean found = false;
                for (int j = 0; j < details.length; j++) {
                    if (details[j].getLoadName().equals(segmentId)) {
                        found = true;
                        break;
                    }
                }
                return !found;
            }
        });
        for (int k = 0; k < listFiles.length; k++) {
            String segmentId = CarbonTablePath.DataFileUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
            if (isCompactionFlow) {
                if (segmentId.contains(".")) {
                    CarbonLoaderUtil.deleteStorePath(listFiles[k].getAbsolutePath());
                }
            } else {
                if (!segmentId.contains(".")) {
                    CarbonLoaderUtil.deleteStorePath(listFiles[k].getAbsolutePath());
                }
            }
        }
    }
}

Example 28 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonUtil method getFilePathExternalFilePath.

public static List<String> getFilePathExternalFilePath(String path) {
    // return the list of carbondata files in the given path.
    CarbonFile segment = FileFactory.getCarbonFile(path, FileFactory.getFileType(path));
    CarbonFile[] dataFiles = segment.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
                return true;
            }
            return false;
        }
    });
    List<String> filePaths = new ArrayList<>(dataFiles.length);
    for (CarbonFile dfiles : dataFiles) {
        filePaths.add(dfiles.getAbsolutePath());
    }
    return filePaths;
}

Example 29 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonUtil method calculateDriverBTreeSize.

/**
 * The method returns the B-Tree for a particular taskId
 *
 * @param taskId
 * @param tableBlockInfoList
 * @param identifier
 */
public static long calculateDriverBTreeSize(String taskId, String bucketNumber, List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier identifier) {
    // need to sort the  block info list based for task in ascending  order so
    // it will be sinkup with block index read from file
    Collections.sort(tableBlockInfoList);
    // geting the index file path
    // TODO need to pass proper partition number when partiton will be supported
    String carbonIndexFilePath = CarbonTablePath.getCarbonIndexFilePath(identifier.getTablePath(), taskId, tableBlockInfoList.get(0).getSegmentId(), bucketNumber, CarbonTablePath.DataFileUtil.getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()), tableBlockInfoList.get(0).getVersion());
    CarbonFile carbonFile = FileFactory.getCarbonFile(carbonIndexFilePath, FileFactory.getFileType(carbonIndexFilePath));
    // in case of carbonIndex file whole file is meta only so reading complete file.
    return carbonFile.getSize();
}

Example 30 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentUpdateStatusManager method getUpdateDeltaFiles.

/**
 * Returns all update delta files of specified Segment.
 *
 * @param segmentId
 * @return
 * @throws Exception
 */
public List<String> getUpdateDeltaFiles(final String segmentId) {
    List<String> updatedDeltaFilesList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    String endTimeStamp = "";
    String startTimeStamp = "";
    String segmentPath = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    for (LoadMetadataDetails eachSeg : segmentDetails) {
        if (eachSeg.getLoadName().equalsIgnoreCase(segmentId)) {
            // if the segment is found then take the start and end time stamp.
            startTimeStamp = eachSeg.getUpdateDeltaStartTimestamp();
            endTimeStamp = eachSeg.getUpdateDeltaEndTimestamp();
        }
    }
    // if start timestamp is empty then no update delta is found. so return empty list.
    if (startTimeStamp.isEmpty()) {
        return updatedDeltaFilesList;
    }
    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimeStamp);
    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimeStamp);
    // else scan the segment for the delta files with the respective timestamp.
    CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathName) {
            String fileName = pathName.getName();
            if (fileName.endsWith(CarbonCommonConstants.UPDATE_DELTA_FILE_EXT)) {
                String firstPart = fileName.substring(0, fileName.indexOf('.'));
                long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1, firstPart.length()));
                if (Long.compare(timestamp, endTimeStampFinal) <= 0 && Long.compare(timestamp, startTimeStampFinal) >= 0) {
                    // if marked for delete then it is invalid.
                    if (!isBlockValid(segmentId, fileName)) {
                        return false;
                    }
                    return true;
                }
            }
            return false;
        }
    });
    for (CarbonFile cfile : files) {
        updatedDeltaFilesList.add(cfile.getCanonicalPath());
    }
    return updatedDeltaFilesList;
}